Blueprint profile integration (#163)

* Integrate Profiles into Blueprints
* Listen for events and deploy blueprints for new enrollments
* Track enrolled status of (re-)enrolling devices and re-issue the Check-In Event under a different topic
* Added references to #149 and #110 for direction/conceptual background
* Closes #149 (largely implemented here)

blueprint/blueprint.go

@@ -1,29 +1,43 @@
 package blueprint
 
 import (
+	"errors"
+
 	"github.com/gogo/protobuf/proto"
 	"github.com/micromdm/micromdm/blueprint/internal/blueprintproto"
 )
 
-type Mobileconfig []byte
+// ApplyAt is a case-insensitive string that specifies at which point the
+// system should apply a Blueprint to devices. For example if a Blueprint has
+// an ApplyAt of "Enroll" then that profile will be applied immediately after
+// a device's enrollment in the MDM system. Currently "Enroll" is the only
+// supported value but more are planned.
+const (
+	ApplyAtEnroll string = "Enroll"
+)
 
 type Blueprint struct {
-	UUID            string         `json:"uuid"`
-	Name            string         `json:"name"`
-	ApplicationURLs []string       `json:"install_application_manifest_urls"`
-	Profiles        []Mobileconfig `json:"profiles"`
+	UUID               string   `json:"uuid"`
+	Name               string   `json:"name"`
+	ApplicationURLs    []string `json:"install_application_manifest_urls"`
+	ProfileIdentifiers []string `json:"profile_ids"`
+	ApplyAt            []string `json:"apply_at"`
+}
+
+func (bp *Blueprint) Verify() error {
+	if bp.Name == "" || bp.UUID == "" {
+		return errors.New("Blueprint must have Name and UUID")
+	}
+	return nil
 }
 
 func MarshalBlueprint(bp *Blueprint) ([]byte, error) {
-	var profiles [][]byte
-	for _, p := range bp.Profiles {
-		profiles = append(profiles, []byte(p))
-	}
 	protobp := blueprintproto.Blueprint{
-		Uuid:          bp.UUID,
-		Name:          bp.Name,
-		ManifestUrls:  bp.ApplicationURLs,
-		Mobileconfigs: profiles,
+		Uuid:         bp.UUID,
+		Name:         bp.Name,
+		ManifestUrls: bp.ApplicationURLs,
+		ProfileIds:   bp.ProfileIdentifiers,
+		ApplyAt:      bp.ApplyAt,
 	}
 	return proto.Marshal(&protobp)
 }
@@ -33,13 +47,10 @@ func UnmarshalBlueprint(data []byte, bp *Blueprint) error {
 	if err := proto.Unmarshal(data, &pb); err != nil {
 		return err
 	}
-	var profiles []Mobileconfig
-	for _, p := range pb.GetMobileconfigs() {
-		profiles = append(profiles, Mobileconfig(p))
-	}
 	bp.UUID = pb.GetUuid()
 	bp.Name = pb.GetName()
 	bp.ApplicationURLs = pb.GetManifestUrls()
-	bp.Profiles = profiles
+	bp.ProfileIdentifiers = pb.GetProfileIds()
+	bp.ApplyAt = pb.GetApplyAt()
 	return nil
 }

blueprint/db.go

@@ -2,9 +2,12 @@ package blueprint
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/boltdb/bolt"
 	"github.com/pkg/errors"
+
+	"github.com/micromdm/micromdm/profile"
 )
 
 const (
@@ -14,9 +17,10 @@ const (
 
 type DB struct {
 	*bolt.DB
+	profDB *profile.DB
 }
 
-func NewDB(db *bolt.DB) (*DB, error) {
+func NewDB(db *bolt.DB, pDB *profile.DB) (*DB, error) {
 	err := db.Update(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(blueprintIndexBucket))
 		if err != nil {
@@ -29,7 +33,8 @@ func NewDB(db *bolt.DB) (*DB, error) {
 		return nil, errors.Wrapf(err, "creating %s bucket", BlueprintBucket)
 	}
 	datastore := &DB{
-		DB: db,
+		DB:     db,
+		profDB: pDB,
 	}
 	return datastore, nil
 }
@@ -64,8 +69,18 @@ func (db *DB) List() ([]Blueprint, error) {
 }
 
 func (db *DB) Save(bp *Blueprint) error {
-	if bp.Name == "" || bp.UUID == "" {
-		return errors.New("cannot Save: blueprint must have Name and UUID")
+	err := bp.Verify()
+	if err != nil {
+		return err
+	}
+	// verify that each Profile ID represents a profile we know about
+	for _, p := range bp.ProfileIdentifiers {
+		if _, err := db.profDB.ProfileById(p); err != nil {
+			if profile.IsNotFound(err) {
+				return errors.New(fmt.Sprintf("Profile ID %s in Blueprint %s does not exist", p, bp.Name))
+			}
+			return errors.Wrap(err, "fetching profile")
+		}
 	}
 	tx, err := db.DB.Begin(true)
 	if err != nil {
@@ -122,6 +137,33 @@ func (db *DB) BlueprintByName(name string) (*Blueprint, error) {
 	return &bp, nil
 }
 
+func (db *DB) BlueprintsByApplyAt(name string) ([]*Blueprint, error) {
+	var bps []*Blueprint
+	err := db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(BlueprintBucket))
+		c := b.Cursor()
+		// TODO: fix this to use an index of ApplyAt strings mapping to
+		// an array of Blueprints or other more efficient means. Looping
+		// over every blueprint is quite inefficient!
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			var bp Blueprint
+			err := UnmarshalBlueprint(v, &bp)
+			if err != nil {
+				fmt.Println("could not Unmarshal Blueprint")
+				continue
+			}
+			for _, n := range bp.ApplyAt {
+				if strings.ToLower(n) == strings.ToLower(name) {
+					bps = append(bps, &bp)
+					break
+				}
+			}
+		}
+		return nil
+	})
+	return bps, err
+}
+
 type notFound struct {
 	ResourceType string
 	Message      string

blueprint/internal/blueprintproto/blueprint.pb.go

@@ -29,10 +29,11 @@ var _ = math.Inf
 const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
 
 type Blueprint struct {
-	Uuid          string   `protobuf:"bytes,1,opt,name=uuid" json:"uuid,omitempty"`
-	Name          string   `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"`
-	ManifestUrls  []string `protobuf:"bytes,3,rep,name=manifest_urls,json=manifestUrls" json:"manifest_urls,omitempty"`
-	Mobileconfigs [][]byte `protobuf:"bytes,4,rep,name=mobileconfigs,proto3" json:"mobileconfigs,omitempty"`
+	Uuid         string   `protobuf:"bytes,1,opt,name=uuid" json:"uuid,omitempty"`
+	Name         string   `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"`
+	ManifestUrls []string `protobuf:"bytes,3,rep,name=manifest_urls,json=manifestUrls" json:"manifest_urls,omitempty"`
+	ProfileIds   []string `protobuf:"bytes,5,rep,name=profile_ids,json=profileIds" json:"profile_ids,omitempty"`
+	ApplyAt      []string `protobuf:"bytes,6,rep,name=apply_at,json=applyAt" json:"apply_at,omitempty"`
 }
 
 func (m *Blueprint) Reset()                    { *m = Blueprint{} }
@@ -61,9 +62,16 @@ func (m *Blueprint) GetManifestUrls() []string {
 	return nil
 }
 
-func (m *Blueprint) GetMobileconfigs() [][]byte {
+func (m *Blueprint) GetProfileIds() []string {
 	if m != nil {
-		return m.Mobileconfigs
+		return m.ProfileIds
+	}
+	return nil
+}
+
+func (m *Blueprint) GetApplyAt() []string {
+	if m != nil {
+		return m.ApplyAt
 	}
 	return nil
 }
@@ -75,14 +83,17 @@ func init() {
 func init() { proto.RegisterFile("blueprint.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 144 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4f, 0xca, 0x29, 0x4d,
-	0x2d, 0x28, 0xca, 0xcc, 0x2b, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x83, 0x0b, 0x80,
-	0xf9, 0x4a, 0x75, 0x5c, 0x9c, 0x4e, 0x30, 0x11, 0x21, 0x21, 0x2e, 0x96, 0xd2, 0xd2, 0xcc, 0x14,
-	0x09, 0x46, 0x05, 0x46, 0x0d, 0xce, 0x20, 0x30, 0x1b, 0x24, 0x96, 0x97, 0x98, 0x9b, 0x2a, 0xc1,
-	0x04, 0x11, 0x03, 0xb1, 0x85, 0x94, 0xb9, 0x78, 0x73, 0x13, 0xf3, 0x32, 0xd3, 0x52, 0x8b, 0x4b,
-	0xe2, 0x4b, 0x8b, 0x72, 0x8a, 0x25, 0x98, 0x15, 0x98, 0x35, 0x38, 0x83, 0x78, 0x60, 0x82, 0xa1,
-	0x45, 0x39, 0xc5, 0x42, 0x2a, 0x5c, 0xbc, 0xb9, 0xf9, 0x49, 0x99, 0x39, 0xa9, 0xc9, 0xf9, 0x79,
-	0x69, 0x99, 0xe9, 0xc5, 0x12, 0x2c, 0x0a, 0xcc, 0x1a, 0x3c, 0x41, 0xa8, 0x82, 0x49, 0x6c, 0x60,
-	0x67, 0x18, 0x03, 0x02, 0x00, 0x00, 0xff, 0xff, 0xc8, 0xef, 0xe9, 0x3b, 0xa9, 0x00, 0x00, 0x00,
+	// 185 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x3c, 0xce, 0x3d, 0x8e, 0xc2, 0x30,
+	0x10, 0x05, 0x60, 0x65, 0xf3, 0xb3, 0xc9, 0xec, 0x06, 0x90, 0x2b, 0x53, 0x11, 0x41, 0x93, 0x8a,
+	0x86, 0x13, 0x40, 0x07, 0x65, 0x24, 0xea, 0xc8, 0x21, 0x0e, 0x1a, 0xc9, 0xb1, 0x2d, 0xff, 0x14,
+	0x1c, 0x89, 0x5b, 0xa2, 0x18, 0x42, 0xf7, 0xde, 0x37, 0x53, 0x3c, 0x58, 0x76, 0xc2, 0x73, 0x6d,
+	0x50, 0xba, 0xbd, 0x36, 0xca, 0x29, 0xb2, 0xf8, 0x42, 0xe8, 0xdb, 0x67, 0x04, 0xc5, 0x69, 0x26,
+	0x42, 0x20, 0xf1, 0x1e, 0x7b, 0x1a, 0x55, 0x51, 0x5d, 0x34, 0x21, 0x4f, 0x26, 0xd9, 0xc8, 0xe9,
+	0xcf, 0xdb, 0xa6, 0x4c, 0x76, 0x50, 0x8e, 0x4c, 0xe2, 0xc0, 0xad, 0x6b, 0xbd, 0x11, 0x96, 0xc6,
+	0x55, 0x5c, 0x17, 0xcd, 0xff, 0x8c, 0x57, 0x23, 0x2c, 0xd9, 0xc0, 0x9f, 0x36, 0x6a, 0x40, 0xc1,
+	0x5b, 0xec, 0x2d, 0x4d, 0xc3, 0x0b, 0x7c, 0xe8, 0xdc, 0x5b, 0xb2, 0x86, 0x9c, 0x69, 0x2d, 0x1e,
+	0x2d, 0x73, 0x34, 0x0b, 0xd7, 0xdf, 0xd0, 0x8f, 0xee, 0x92, 0xe4, 0xc9, 0x2a, 0x6d, 0xca, 0x51,
+	0x75, 0x28, 0xf8, 0x4d, 0xc9, 0x01, 0xef, 0xb6, 0xcb, 0xc2, 0xe4, 0xc3, 0x2b, 0x00, 0x00, 0xff,
+	0xff, 0x1d, 0x05, 0x14, 0x9a, 0xd5, 0x00, 0x00, 0x00,
 }

blueprint/internal/blueprintproto/blueprint.proto

@@ -3,8 +3,12 @@ syntax = "proto3";
 package blueprintproto;
 
 message Blueprint {
+	reserved 4;
+	reserved "mobileconfigs";
+
 	string uuid = 1;
 	string name = 2;
 	repeated string manifest_urls = 3;
-	repeated bytes mobileconfigs = 4;
+	repeated string profile_ids = 5;
+	repeated string apply_at = 6;
 }

blueprint/listener.go

@@ -0,0 +1,116 @@
+package blueprint
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/micromdm/mdm"
+	"github.com/micromdm/micromdm/checkin"
+	"github.com/micromdm/micromdm/command"
+	"github.com/micromdm/micromdm/device"
+	"github.com/micromdm/micromdm/profile"
+	"github.com/micromdm/micromdm/pubsub"
+)
+
+func (db *DB) ApplyToDevice(ctx context.Context, svc command.Service, bp *Blueprint, udid string) error {
+
+	var requests []*mdm.CommandRequest
+	for _, appURL := range bp.ApplicationURLs {
+		requests = append(requests, &mdm.CommandRequest{
+			RequestType: "InstallApplication",
+			UDID:        udid,
+			InstallApplication: mdm.InstallApplication{
+				ManifestURL:     appURL,
+				ManagementFlags: 1,
+			},
+		})
+	}
+
+	for _, p := range bp.ProfileIdentifiers {
+		foundProfile, err := db.profDB.ProfileById(p)
+		if err != nil {
+			if profile.IsNotFound(err) {
+				fmt.Printf("Profile ID %s in Blueprint %s does not exist\n", p, bp.Name)
+				continue
+			}
+			fmt.Println(err)
+			continue
+		}
+
+		requests = append(requests, &mdm.CommandRequest{
+			RequestType: "InstallProfile",
+			UDID:        udid,
+			InstallProfile: mdm.InstallProfile{
+				Payload: foundProfile.Mobileconfig,
+			},
+		})
+	}
+
+	for _, r := range requests {
+		_, err := svc.NewCommand(ctx, r)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (db *DB) StartListener(sub pubsub.Subscriber, cmdSvc command.Service) error {
+	tokenUpdateEvents, err := sub.Subscribe("applyAtEnroll", device.DeviceEnrolledTopic)
+	if err != nil {
+		return errors.Wrapf(err,
+			"subscribing devices to %s topic", device.DeviceEnrolledTopic)
+	}
+
+	go func() {
+		for {
+			select {
+			case event := <-tokenUpdateEvents:
+				var ev checkin.Event
+				if err := checkin.UnmarshalEvent(event.Message, &ev); err != nil {
+					fmt.Println(err)
+					continue
+				}
+				if ev.Command.UserID != "" {
+					// skip UserID token updates
+					continue
+				}
+				bps, err := db.BlueprintsByApplyAt(ApplyAtEnroll)
+				if err != nil {
+					fmt.Println(err)
+					continue
+				}
+				ctx := context.Background()
+				for _, bp := range bps {
+					fmt.Printf("applying blueprint %s to %s\n", bp.Name, ev.Command.UDID)
+					err := db.ApplyToDevice(ctx, cmdSvc, bp, ev.Command.UDID)
+					if err != nil {
+						fmt.Println(err)
+					}
+				}
+
+				if ev.Command.AwaitingConfiguration {
+					_, err := cmdSvc.NewCommand(ctx, &mdm.CommandRequest{
+						RequestType: "DeviceConfigured",
+						UDID:        ev.Command.UDID,
+					})
+					if err != nil {
+						fmt.Println(errors.Wrapf(err, "sending DeviceConfigured"))
+					}
+				}
+
+				// TODO: See notes from here:
+				// https://github.com/jessepeterson/micromdm/blob/8b068ac98d06954bb3e08b1557c193007932552b/blueprint/listener.go#L73-L103
+				// Also see discussion here for general direction:
+				// https://github.com/micromdm/micromdm/pull/149
+				// Finally see discussion here for high-level goals:
+				// https://github.com/micromdm/micromdm/issues/110
+			}
+
+		}
+	}()
+
+	return nil
+}

cmd/mdmctl/apply.go

@@ -125,7 +125,7 @@ func (cmd *applyCommand) applyBlueprint(args []string) error {
 		newBlueprint.Name = "exampleName"
 		newBlueprint.UUID = uuid.NewV4().String()
 		newBlueprint.ApplicationURLs = []string{cmd.config.ServerURL + "repo/exampleAppManifest.plist"}
-		newBlueprint.Profiles = []blueprint.Mobileconfig{blueprint.Mobileconfig([]byte("this should be a configuration profile"))}
+		newBlueprint.ProfileIdentifiers = []string{"com.example.my.profile"}
 
 		enc := json.NewEncoder(newBlueprintFile)
 		enc.SetIndent("", "  ")

cmd/mdmctl/get.go

@@ -201,15 +201,22 @@ func (cmd *getCommand) getBlueprints(args []string) error {
 	}
 
 	w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0)
-	fmt.Fprintf(w, "Name\tUUID\tManifests\tProfiles\n")
+	fmt.Fprintf(w, "Name\tUUID\tManifests\tProfiles\tApply At\n")
 	for _, bp := range blueprints {
+		var applyAtStr string
+		if len(bp.ApplyAt) > 0 {
+			applyAtStr = strings.Join(bp.ApplyAt, ",")
+		} else {
+			applyAtStr = "(None)"
+		}
 		fmt.Fprintf(
 			w,
-			"%s\t%s\t%d\t%d\n",
+			"%s\t%s\t%d\t%d\t%s\n",
 			bp.Name,
 			bp.UUID,
 			len(bp.ApplicationURLs),
-			len(bp.Profiles),
+			len(bp.ProfileIdentifiers),
+			applyAtStr,
 		)
 	}
 	w.Flush()

device/db.go

@@ -18,13 +18,15 @@ const (
 	// The deviceIndexBucket index bucket stores serial number and UDID references
 	// to the device uuid.
 	deviceIndexBucket = "mdm.DeviceIdx"
+
+	DeviceEnrolledTopic = "mdm.DeviceEnrolled"
 )
 
 type DB struct {
 	*bolt.DB
 }
 
-func NewDB(db *bolt.DB, sub pubsub.Subscriber) (*DB, error) {
+func NewDB(db *bolt.DB, pubsubSvc pubsub.PublishSubscriber) (*DB, error) {
 	err := db.Update(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(deviceIndexBucket))
 		if err != nil {
@@ -39,10 +41,10 @@ func NewDB(db *bolt.DB, sub pubsub.Subscriber) (*DB, error) {
 	datastore := &DB{
 		DB: db,
 	}
-	if sub == nil { // don't start the poller without pubsub.
+	if pubsubSvc == nil { // don't start the poller without pubsub.
 		return datastore, nil
 	}
-	if err := datastore.pollCheckin(sub); err != nil {
+	if err := datastore.pollCheckin(pubsubSvc); err != nil {
 		return nil, err
 	}
 	return datastore, nil
@@ -162,23 +164,23 @@ func isNotFound(err error) bool {
 	return false
 }
 
-func (db *DB) pollCheckin(sub pubsub.Subscriber) error {
-	authenticateEvents, err := sub.Subscribe("devices", checkin.AuthenticateTopic)
+func (db *DB) pollCheckin(pubsubSvc pubsub.PublishSubscriber) error {
+	authenticateEvents, err := pubsubSvc.Subscribe("devices", checkin.AuthenticateTopic)
 	if err != nil {
 		return errors.Wrapf(err,
 			"subscribing devices to %s topic", checkin.AuthenticateTopic)
 	}
-	tokenUpdateEvents, err := sub.Subscribe("devices", checkin.TokenUpdateTopic)
+	tokenUpdateEvents, err := pubsubSvc.Subscribe("devices", checkin.TokenUpdateTopic)
 	if err != nil {
 		return errors.Wrapf(err,
 			"subscribing devices to %s topic", checkin.TokenUpdateTopic)
 	}
-	checkoutEvents, err := sub.Subscribe("devices", checkin.CheckoutTopic)
+	checkoutEvents, err := pubsubSvc.Subscribe("devices", checkin.CheckoutTopic)
 	if err != nil {
 		return errors.Wrapf(err,
 			"subscribing devices to %s topic", checkin.CheckoutTopic)
 	}
-	depSyncEvents, err := sub.Subscribe("devices", depsync.SyncTopic)
+	depSyncEvents, err := pubsubSvc.Subscribe("devices", depsync.SyncTopic)
 	if err != nil {
 		return errors.Wrapf(err,
 			"subscribing devices to %s topic", depsync.SyncTopic)
@@ -209,6 +211,7 @@ func (db *DB) pollCheckin(sub pubsub.Subscriber) error {
 					continue
 				} else if err == nil {
 					fmt.Printf("re-enrolling device %s\n", ev.Command.SerialNumber)
+					newDevice.Enrolled = false
 				}
 
 				// only create new UUID on initial enrollment.
@@ -250,11 +253,23 @@ func (db *DB) pollCheckin(sub pubsub.Subscriber) error {
 				dev.PushMagic = ev.Command.PushMagic
 				dev.UnlockToken = ev.Command.UnlockToken.String()
 				dev.AwaitingConfiguration = ev.Command.AwaitingConfiguration
-				dev.Enrolled = true
+				dev.LastCheckin = time.Now()
+				var newlyEnrolled bool = false
+				if dev.Enrolled == false {
+					newlyEnrolled = true
+					dev.Enrolled = true
+				}
 				if err := db.Save(dev); err != nil {
 					fmt.Println(err)
 					continue
 				}
+				if newlyEnrolled {
+					fmt.Printf("device %s enrolled\n", ev.Command.UDID)
+					err := pubsubSvc.Publish(DeviceEnrolledTopic, event.Message)
+					if err != nil {
+						fmt.Println(err)
+					}
+				}
 			case event := <-depSyncEvents:
 				var ev depsync.Event
 				if err := depsync.UnmarshalEvent(event.Message, &ev); err != nil {

profile/db.go

@@ -92,7 +92,7 @@ func (e *notFound) Error() string {
 	return fmt.Sprintf("not found: %s %s", e.ResourceType, e.Message)
 }
 
-func isNotFound(err error) bool {
+func IsNotFound(err error) bool {
 	if _, ok := err.(*notFound); ok {
 		return true
 	}

serve.go

@@ -147,16 +147,20 @@ func serve(args []string) error {
 		stdlog.Fatal(err)
 	}
 
-	bpDB, err := blueprint.NewDB(sm.db)
+	profDB, err := profile.NewDB(sm.db)
 	if err != nil {
 		stdlog.Fatal(err)
 	}
 
-	profDB, err := profile.NewDB(sm.db)
+	bpDB, err := blueprint.NewDB(sm.db, profDB)
 	if err != nil {
 		stdlog.Fatal(err)
 	}
 
+	if err := bpDB.StartListener(sm.pubclient, sm.commandService); err != nil {
+		stdlog.Fatal(err)
+	}
+
 	ctx := context.Background()
 	httpLogger := log.With(logger, "transport", "http")
 	var checkinEndpoint endpoint.Endpoint