diff --git a/CHANGELOG.md b/CHANGELOG.md index f1141cba..fa7979cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## [Unreleased](https://github.com/micromdm/micromdm/compare/v1.10.1...main) - Add SoftwareUpdateSettings to Settings command (#771, #856) +- Ensure errors are logged on the checkin and connect endpoints (#871) +- Add support for raw plist commands (#864) ## [v1.10.1](https://github.com/micromdm/micromdm/compare/v1.10.0...v1.10.1) January 24, 2023 diff --git a/docs/user-guide/api-and-webhooks.md b/docs/user-guide/api-and-webhooks.md index e64eae26..524683cd 100644 --- a/docs/user-guide/api-and-webhooks.md +++ b/docs/user-guide/api-and-webhooks.md @@ -212,6 +212,7 @@ For example, to install a configuration profile to a device, you can schedule th ``` POST /v1/commands HTTP/1.1 Authorization: Basic bWljcm9tZG06c3VwZXJzZWNyZXQ= + { "udid": "55693EB3-DF03-5FD1-9263-F7CDB8AD7FFD", "request_type": "InstallProfile", @@ -220,3 +221,53 @@ Authorization: Basic bWljcm9tZG06c3VwZXJzZWNyZXQ= ``` MicroMDM will convert this request into a complete command, and schedule it on the queue. It will then send a push notification to ask the device to check in, and respond with the InstallProfile command. + +# Schedule Raw Commands with the API + +[PR #864](https://github.com/micromdm/micromdm/pull/864) added support for queuing raw plist commands. This is useful for queuing commands that aren't currently supported (e.g. missing commands or missing fields) by MicroMDM and can also help migrate to NanoMDM. + +The raw command endpoint differs from the main command endpoint in the following ways: + +* The MDM topic (e.g. the device UDID) is specified in the URL +* The body is the raw command plist instead of MicroMDM's JSON schema +* The command is not validated (the body is only checked to be a valid plist) + +Here's an example [ProfileList](https://developer.apple.com/documentation/devicemanagement/list_the_installed_profiles) command: + +``` +POST /v1/commands/55693EB3-DF03-5FD1-9263-F7CDB8AD7FFD HTTP/1.1 +Authorization: Basic bWljcm9tZG06c3VwZXJzZWNyZXQ= + + + + + + Command + + ManagedOnly + + RequestType + ProfileList + + CommandUUID + 0001_ProfileList + + +``` + +A helper script is also available at `./tools/api/raw_command`: + +`$ ./raw_command 55693EB3-DF03-5FD1-9263-F7CDB8AD7FFD path/to/cmd.plist` + +## Clearing the Command Queue + +Since the raw command api endpoint doesn't validate the body as a valid command, it's possible to queue a malformed command, which may cause the client device to stop processing commands from the queue. [PR #864](https://github.com/micromdm/micromdm/pull/864) also includes an endpoint to clear the command queue for a device: + +``` +DELETE /v1/commands/ HTTP/1.1 +Authorization: Basic bWljcm9tZG06c3VwZXJzZWNyZXQ= +``` + +A helper script is also available at `./tools/api/clear_queue`: + +`$ ./clear_queue 55693EB3-DF03-5FD1-9263-F7CDB8AD7FFD` diff --git a/platform/command/clear.go b/platform/command/clear.go new file mode 100644 index 00000000..5d9eb7b6 --- /dev/null +++ b/platform/command/clear.go @@ -0,0 +1,47 @@ +package command + +import ( + "context" + "net/http" + + "github.com/go-kit/kit/endpoint" + "github.com/gorilla/mux" + "github.com/micromdm/micromdm/mdm" + "github.com/pkg/errors" +) + +func (svc *CommandService) ClearQueue(ctx context.Context, udid string) error { + if err := svc.queue.Clear(ctx, mdm.CheckinEvent{Command: mdm.CheckinCommand{UDID: udid}}); err != nil { + return errors.Wrap(err, "clearing command queue") + } + return nil +} + +type clearRequest struct { + UDID string +} + +type clearResponse struct { + Err error `json:"error,omitempty"` +} + +func (r clearResponse) Failed() error { return r.Err } +func (r clearResponse) StatusCode() int { return http.StatusOK } + +func decodeClearRequest(ctx context.Context, r *http.Request) (interface{}, error) { + return clearRequest{UDID: mux.Vars(r)["udid"]}, nil +} + +// MakeClearQueueEndpoint creates an endpoint which clears device queues. +func MakeClearQueueEndpoint(svc Service) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(clearRequest) + if req.UDID == "" { + return clearResponse{Err: errEmptyRequest}, nil + } + if err := svc.ClearQueue(ctx, req.UDID); err != nil { + return clearResponse{Err: err}, nil + } + return clearResponse{}, nil + } +} diff --git a/platform/command/event.go b/platform/command/event.go index 2afabadf..240fbd8a 100644 --- a/platform/command/event.go +++ b/platform/command/event.go @@ -61,3 +61,45 @@ func UnmarshalEvent(data []byte, e *Event) error { e.Payload = &payload return nil } + +type RawEvent struct { + CommandUUID string + Time time.Time + DeviceUDID string + Payload []byte +} + +// NewRawEvent returns a RawEvent with the current time. +func NewRawEvent(cmd *RawCommand) *RawEvent { + event := RawEvent{ + CommandUUID: cmd.CommandUUID, + Time: time.Now().UTC(), + DeviceUDID: cmd.UDID, + Payload: cmd.Raw, + } + return &event +} + +// MarshalRawEvent serializes a RawEvent to a protocol buffer wire format. +func MarshalRawEvent(e *RawEvent) ([]byte, error) { + return proto.Marshal(&commandproto.Event{ + Id: e.CommandUUID, // Id isn't used anywhere, so it's repurposed for CommandUUID + Time: e.Time.UnixNano(), + DeviceUdid: e.DeviceUDID, + PayloadBytes: e.Payload, + }) +} + +// UnmarshalRawEvent parses a protocol buffer representation of data into +// the RawEvent. +func UnmarshalRawEvent(data []byte, e *RawEvent) error { + var pb commandproto.Event + if err := proto.Unmarshal(data, &pb); err != nil { + return errors.Wrap(err, "unmarshal pb Event") + } + e.CommandUUID = pb.Id + e.Time = time.Unix(0, pb.Time).UTC() + e.DeviceUDID = pb.DeviceUdid + e.Payload = pb.PayloadBytes + return nil +} diff --git a/platform/command/event_test.go b/platform/command/event_test.go new file mode 100644 index 00000000..a38f4dff --- /dev/null +++ b/platform/command/event_test.go @@ -0,0 +1,49 @@ +package command_test + +import ( + "reflect" + "testing" + + "github.com/micromdm/micromdm/platform/command" +) + +const testRawCmd = ` + + + + Command + + ManagedOnly + + RequestType + ProfileList + + CommandUUID + 0001_ProfileList + +` + +func TestRawEvent(t *testing.T) { + cmd := &command.RawCommand{ + UDID: "1234", + CommandUUID: "0001_ProfileList", + Raw: []byte(testRawCmd), + } + cmd.Command.RequestType = "ProfileList" + + ev := command.NewRawEvent(cmd) + + buf, err := command.MarshalRawEvent(ev) + if err != nil { + t.Fatalf("could not marshal event: %v", err) + } + + ev2 := new(command.RawEvent) + if err = command.UnmarshalRawEvent(buf, ev2); err != nil { + t.Fatalf("could not unmarshal event: %v", err) + } + + if !reflect.DeepEqual(ev, ev2) { + t.Error("expected events to be equal") + } +} diff --git a/platform/command/new_command.go b/platform/command/new_command.go index 6d880e59..26e68626 100644 --- a/platform/command/new_command.go +++ b/platform/command/new_command.go @@ -1,9 +1,14 @@ package command import ( + "bytes" + "fmt" + "io" "net/http" "github.com/go-kit/kit/endpoint" + "github.com/gorilla/mux" + "github.com/groob/plist" "github.com/pkg/errors" "golang.org/x/net/context" @@ -14,6 +19,9 @@ import ( const ( // CommandTopic is a PubSub topic that events are published to. CommandTopic = "mdm.Command" + + // RawCommandTopic is a PubSub topic that events are published to. + RawCommandTopic = "mdm.RawCommand" ) func (svc *CommandService) NewCommand(ctx context.Context, request *mdm.CommandRequest) (*mdm.CommandPayload, error) { @@ -35,6 +43,21 @@ func (svc *CommandService) NewCommand(ctx context.Context, request *mdm.CommandR return payload, nil } +func (svc *CommandService) NewRawCommand(ctx context.Context, cmd *RawCommand) error { + if cmd == nil { + return errors.New("empty RawCommand") + } + event := NewRawEvent(cmd) + msg, err := MarshalRawEvent(event) + if err != nil { + return errors.Wrap(err, "marshalling raw mdm command event") + } + if err := svc.publisher.Publish(context.TODO(), RawCommandTopic, msg); err != nil { + return errors.Wrapf(err, "publish raw mdm command on topic: %s", RawCommandTopic) + } + return nil +} + type newCommandRequest struct { mdm.CommandRequest } @@ -69,3 +92,65 @@ func MakeNewCommandEndpoint(svc Service) endpoint.Endpoint { return newCommandResponse{Payload: payload}, nil } } + +type RawCommand struct { + UDID string `json:"udid" plist:"-"` + CommandUUID string `json:"command_uuid"` + Command struct { + RequestType string `json:"request_type"` + } `json:"command"` + Raw []byte `plist:"-" json:"payload"` +} + +type newRawCommandRequest struct { + RawCommand +} + +type newRawCommandResponse struct { + Payload *RawCommand `json:"payload,omitempty"` + Err error `json:"error,omitempty"` +} + +func (r newRawCommandResponse) Failed() error { return r.Err } +func (r newRawCommandResponse) StatusCode() int { return http.StatusCreated } + +func decodeNewRawCommandRequest(ctx context.Context, r *http.Request) (interface{}, error) { + udid, ok := mux.Vars(r)["udid"] + if !ok { + return nil, errors.New("empty udid") + } + + payload, err := io.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("read payload body: %w", err) + } + + // verify body is valid plist and parse CommandUUID and RequestType + var req newRawCommandRequest + if err := plist.NewXMLDecoder(bytes.NewBuffer(payload)).Decode(&req); err != nil { + return nil, fmt.Errorf("parse payload as plist: %w", err) + } + + req.UDID = udid + req.Raw = payload + return req, nil +} + +var errMalformedRequest = errors.New("request is malformed") + +// MakeNewRawCommandEndpoint creates an endpoint which creates new raw MDM Commands. +func MakeNewRawCommandEndpoint(svc Service) endpoint.Endpoint { + return func(ctx context.Context, request interface{}) (interface{}, error) { + req := request.(newRawCommandRequest) + if req.UDID == "" { + return newRawCommandResponse{Err: errEmptyRequest}, nil + } + if req.CommandUUID == "" || req.Command.RequestType == "" { + return newRawCommandResponse{Err: errMalformedRequest}, nil + } + if err := svc.NewRawCommand(ctx, &req.RawCommand); err != nil { + return newRawCommandResponse{Err: err}, nil + } + return newRawCommandResponse{Payload: &req.RawCommand}, nil + } +} diff --git a/platform/command/server.go b/platform/command/server.go index ff6a63fd..35d719a0 100644 --- a/platform/command/server.go +++ b/platform/command/server.go @@ -8,22 +8,41 @@ import ( ) type Endpoints struct { - NewCommandEndpoint endpoint.Endpoint + NewCommandEndpoint endpoint.Endpoint + NewRawCommandEndpoint endpoint.Endpoint + ClearQueueEndpoint endpoint.Endpoint } func MakeServerEndpoints(s Service, outer endpoint.Middleware, others ...endpoint.Middleware) Endpoints { return Endpoints{ - NewCommandEndpoint: endpoint.Chain(outer, others...)(MakeNewCommandEndpoint(s)), + NewCommandEndpoint: endpoint.Chain(outer, others...)(MakeNewCommandEndpoint(s)), + NewRawCommandEndpoint: endpoint.Chain(outer, others...)(MakeNewRawCommandEndpoint(s)), + ClearQueueEndpoint: endpoint.Chain(outer, others...)(MakeClearQueueEndpoint(s)), } } func RegisterHTTPHandlers(r *mux.Router, e Endpoints, options ...httptransport.ServerOption) { // POST /v1/commands Add new MDM Command to device queue. - r.Methods("POST").Path("/v1/commands").Handler(httptransport.NewServer( e.NewCommandEndpoint, decodeNewCommandRequest, httputil.EncodeJSONResponse, options..., )) + + // POST /v1/commands/udid Add new MDM Command with raw plist to device queue. + r.Methods("POST").Path("/v1/commands/{udid}").Handler(httptransport.NewServer( + e.NewRawCommandEndpoint, + decodeNewRawCommandRequest, + httputil.EncodeJSONResponse, + options..., + )) + + // DELETE /v1/commands/udid Clear device queue. + r.Methods("DELETE").Path("/v1/commands/{udid}").Handler(httptransport.NewServer( + e.ClearQueueEndpoint, + decodeClearRequest, + httputil.EncodeJSONResponse, + options..., + )) } diff --git a/platform/command/service.go b/platform/command/service.go index cf297432..8e2620a7 100644 --- a/platform/command/service.go +++ b/platform/command/service.go @@ -2,6 +2,7 @@ package command import ( + mdmsvc "github.com/micromdm/micromdm/mdm" "github.com/micromdm/micromdm/mdm/mdm" "github.com/micromdm/micromdm/platform/pubsub" "golang.org/x/net/context" @@ -9,15 +10,24 @@ import ( type Service interface { NewCommand(context.Context, *mdm.CommandRequest) (*mdm.CommandPayload, error) + NewRawCommand(context.Context, *RawCommand) error + ClearQueue(ctx context.Context, udid string) error +} + +// Queue is an MDM Command Queue. +type Queue interface { + Clear(context.Context, mdmsvc.CheckinEvent) error } type CommandService struct { publisher pubsub.Publisher + queue Queue } -func New(pub pubsub.Publisher) (*CommandService, error) { +func New(pub pubsub.Publisher, queue Queue) (*CommandService, error) { svc := CommandService{ publisher: pub, + queue: queue, } return &svc, nil } diff --git a/platform/queue/inmem/inmem.go b/platform/queue/inmem/inmem.go index 53b6f546..ee761161 100644 --- a/platform/queue/inmem/inmem.go +++ b/platform/queue/inmem/inmem.go @@ -33,6 +33,7 @@ func New(pubsub pubsub.PublishSubscriber, logger log.Logger) *QueueInMem { queue: make(map[string]*list.List), } q.startPolling(pubsub) + q.startRawPolling(pubsub) return q } @@ -169,3 +170,44 @@ func (q *QueueInMem) startPolling(pubsub pubsub.PublishSubscriber) error { }() return nil } + +func (q *QueueInMem) startRawPolling(pubsub pubsub.PublishSubscriber) error { + events, err := pubsub.Subscribe(context.TODO(), "command-queue", command.RawCommandTopic) + if err != nil { + return err + } + go func() { + for { + select { + case event := <-events: + var cmdEvent command.RawEvent + if err := command.UnmarshalRawEvent(event.Message, &cmdEvent); err != nil { + level.Info(q.logger).Log( + "msg", "unmarshal command event from pubsub", + "err", err, + ) + continue + } + q.enqueue( + q.getList(cmdEvent.DeviceUDID), + cmdEvent.CommandUUID, + cmdEvent.Payload, + ) + level.Info(q.logger).Log( + "msg", "queued raw command for device", + "device_udid", cmdEvent.DeviceUDID, + "command_uuid", cmdEvent.CommandUUID, + ) + + err = boltqueue.PublishCommandQueued(pubsub, cmdEvent.DeviceUDID, cmdEvent.CommandUUID) + if err != nil { + level.Info(q.logger).Log( + "msg", "publish command to queued topic", + "err", err, + ) + } + } + } + }() + return nil +} diff --git a/platform/queue/queue.go b/platform/queue/queue.go index d7bfa8e5..45913d4a 100644 --- a/platform/queue/queue.go +++ b/platform/queue/queue.go @@ -212,6 +212,10 @@ func NewQueue(db *bolt.DB, pubsub pubsub.PublishSubscriber, opts ...Option) (*St return nil, err } + if err := datastore.pollRawCommands(pubsub); err != nil { + return nil, err + } + return datastore, nil } @@ -318,6 +322,58 @@ func (db *Store) pollCommands(pubsub pubsub.PublishSubscriber) error { return nil } +func (db *Store) pollRawCommands(pubsub pubsub.PublishSubscriber) error { + commandEvents, err := pubsub.Subscribe(context.TODO(), "command-queue", command.RawCommandTopic) + if err != nil { + return errors.Wrapf(err, + "subscribing push to %s topic", command.RawCommandTopic) + } + go func() { + for { + select { + case event := <-commandEvents: + var ev command.RawEvent + if err := command.UnmarshalRawEvent(event.Message, &ev); err != nil { + level.Info(db.logger).Log("msg", "unmarshal raw command event in queue", "err", err) + continue + } + + cmd := new(DeviceCommand) + cmd.DeviceUDID = ev.DeviceUDID + byUDID, err := db.DeviceCommand(ev.DeviceUDID) + if err == nil && byUDID != nil { + cmd = byUDID + } + newCmd := Command{ + UUID: ev.CommandUUID, + Payload: ev.Payload, + } + cmd.Commands = append(cmd.Commands, newCmd) + if err := db.Save(cmd); err != nil { + level.Info(db.logger).Log("msg", "save command in db", "err", err) + continue + } + level.Info(db.logger).Log( + "msg", "queued raw event for device", + "device_udid", ev.DeviceUDID, + "command_uuid", ev.CommandUUID, + ) + + err = PublishCommandQueued(pubsub, ev.DeviceUDID, ev.CommandUUID) + if err != nil { + level.Info(db.logger).Log( + "msg", "publish command to queued topic", + "err", err, + ) + continue + } + } + } + }() + + return nil +} + func isNotFound(err error) bool { if _, ok := err.(*notFound); ok { return true diff --git a/server/server.go b/server/server.go index dd4be7ec..976a7528 100644 --- a/server/server.go +++ b/server/server.go @@ -74,6 +74,8 @@ type Server struct { SCEPService scep.Service ConfigService config.Service + CommandQueue mdm.Queue + WebhooksHTTPClient *http.Client } @@ -102,10 +104,6 @@ func (c *Server) Setup(logger log.Logger) error { return err } - if err := c.setupCommandService(); err != nil { - return err - } - if err := c.setupWebhooks(logger); err != nil { return err } @@ -114,6 +112,10 @@ func (c *Server) Setup(logger log.Logger) error { return err } + if err := c.setupCommandService(); err != nil { + return err + } + if err := c.setupDepClient(); err != nil { return err } @@ -162,7 +164,7 @@ func (c *Server) setupRemoveService() error { } func (c *Server) setupCommandService() error { - commandService, err := command.New(c.PubClient) + commandService, err := command.New(c.PubClient, c.CommandQueue) if err != nil { return err } @@ -191,6 +193,8 @@ func (c *Server) setupCommandQueue(logger log.Logger) error { return fmt.Errorf("invalid command queue type: %s", c.Queue) } + c.CommandQueue = q + devDB, err := devicebuiltin.NewDB(c.DB) if err != nil { return errors.Wrap(err, "new device db") diff --git a/tools/api/clear_queue b/tools/api/clear_queue new file mode 100755 index 00000000..9b4fd4a2 --- /dev/null +++ b/tools/api/clear_queue @@ -0,0 +1,5 @@ +#!/bin/bash +source $MICROMDM_ENV_PATH +endpoint="v1/commands/$1" + +curl $CURL_OPTS -K <(cat <<< "-u micromdm:$API_TOKEN") -X DELETE "$SERVER_URL/$endpoint" diff --git a/tools/api/raw_command b/tools/api/raw_command new file mode 100755 index 00000000..dbf75e21 --- /dev/null +++ b/tools/api/raw_command @@ -0,0 +1,6 @@ +#!/bin/bash +# raw_cmd $udid $path/to/cmd.plist +source $MICROMDM_ENV_PATH +endpoint="v1/commands/$1" + +curl $CURL_OPTS -K <(cat <<< "-u micromdm:$API_TOKEN") --data "@$2" "$SERVER_URL/$endpoint"