Play around with this:
myLayer = app.project.activeItem.selectedLayers[0];
ratio = .5523;
h = myLayer.width/2;
v = myLayer.height/2;
th = h*ratio;
tv = v*ratio;
newMask = myLayer.Masks.addProperty("ADBE Mask Atom");
newMask.maskMode = MaskMode.ADD;
myProperty = newMask.property("ADBE Mask Shape");
myShape = myProperty.value;
myShape.vertices = [[h,0],[0,v],[h,2*v],[2*h,v]];
myShape.inTangents = [[th,0],[0,-tv],[-th,0],[0,tv]];
myShape.outTangents = [[-th,0],[0,tv],[th,0],[0,-tv]];
myShape.closed = true;
myProperty.setValue(myShape);
Dan