installed_certificates #14 plus conflict resolution (#21)

* Stub new method for application datastore
Handle command failures more gracefully than continuously retrying. Failures are not logged
Add table for certificates.

(cherry picked from commit 9f580a6)

* Remove methods with signature NewDatastore() from datastores and injection of the logger instance into each datastore because groob is a doodoo head :)
Move NewDatastore methods into unit tests
Reinstate NewDB test
Create certificates package including a Certificate type and a datastore.
Began working on a method ackCertificateList in the connect service.
Boilerplate for management service test.
Renamed some tables in certificates migration, hopefully nobody notices.
Applications response handler should be omitted from this branch.

(cherry picked from commit 8bc2fc7)

* Add device uuid to certificate, as there will never be a normalised form of the certificate data.
Change statments to reflect table change from certificates to devices_certificates
Add the CertificateList request type to the connect service's Acknowledge method
Add certificates datastore to the management service, and add endpoints and request/responses for retrieving certificates by device uuid to the management endpoujnt.
Kludge. commits regarding app service will be amended.
(cherry picked from commit c723c80)

* Don't forget the endpoint definition for certificates!

(cherry picked from commit 4e5fc1f)

* Use byte field for certificate data.

(cherry picked from commit 66d5036)

* Fix empty import in certificates

(cherry picked from commit b6d4dca)

* Add table for certificates.

(cherry picked from commit 06bc231)

* Renamed some tables in certificates migration, hopefully nobody notices.
Rebase develop onto master

(cherry picked from commit 775484d)

* Certificate list responses are saved via replacing the entire certificate list on a per device basis. This is because neither the common name nor the data could be used as a unique constraint in the certificates table.
Few small changes to imports/style.

(cherry picked from commit e0a63e1)

* Properly rollback if certificate insert fails for any certificate in a response.

(cherry picked from commit 99e1ea1)

* Fix several incorrect statements and struct tags which were preventing the certificates management endpoint from listing certs.
Certificate listing is working without a base64 encoded representation of each certificate.

(cherry picked from commit 685dfef)

* Add commands index to command datastore.
Add simple test for commands index
Add commands index endpoint

(cherry picked from commit 4e2a2a2)

* Add handler for GET /mdm/commands

(cherry picked from commit ae71b8f)

* Added Find() method to commands datastore so that the request that matches a response can be retrieved by the connect service.
Added Find() method to command service

(cherry picked from commit ebceb28)

* Fix globally scoped vars in command datastore test suite

* groob prefers inline definition of struct members.

* Uppercase CommandUuid

certificates/certificate.go

@@ -0,0 +1,9 @@
+package certificates
+
+type Certificate struct {
+	UUID       string `db:"certificate_uuid" json:"uuid"`
+	DeviceUUID string `db:"device_uuid" json:"device_uuid"`
+	Data       []byte `db:"data" json:"data,omitempty"`
+	CommonName string `db:"common_name" json:"common_name,omitempty"`
+	IsIdentity bool   `db:"is_identity" json:"is_identity"`
+}

certificates/datastore.go

@@ -0,0 +1,165 @@
+package certificates
+
+import (
+	"fmt"
+	kitlog "github.com/go-kit/kit/log"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/lib/pq" // postgres driver
+	"github.com/pkg/errors"
+	"strings"
+	"time"
+)
+
+var (
+	insertCertificateStmt = `INSERT INTO devices_certificates (
+		device_uuid,
+		common_name,
+		data,
+		is_identity
+	) VALUES ($1, $2, $3, $4)
+	RETURNING certificate_uuid;`
+
+	selectCertificatesStmt = `SELECT
+		certificate_uuid,
+		device_uuid,
+		common_name,
+		data,
+		is_identity
+		FROM devices_certificates`
+
+	selectCertificatesByDeviceUdidStmt = `SELECT
+		certificate_uuid,
+		devices_certificates.device_uuid device_uuid,
+		common_name,
+		data,
+		is_identity
+		FROM devices_certificates
+		INNER JOIN devices ON devices_certificates.device_uuid = devices.device_uuid
+		WHERE devices.udid = $1`
+
+	selectCertificatesByDeviceUuidStmt = `SELECT
+		certificate_uuid,
+		devices_certificates.device_uuid device_uuid,
+		common_name,
+		data,
+		is_identity
+		FROM devices_certificates
+		INNER JOIN devices ON devices_certificates.device_uuid = devices.device_uuid
+		WHERE devices.device_uuid = $1`
+)
+
+// This Datastore manages a list of certificates assigned to devices.
+type Datastore interface {
+	New(crt *Certificate) (string, error)
+	Certificates(params ...interface{}) ([]Certificate, error)
+	GetCertificatesByDeviceUDID(udid string) ([]Certificate, error)
+	GetCertificatesByDeviceUUID(uuid string) ([]Certificate, error)
+	ReplaceCertificatesByDeviceUUID(uuid string, certificates []Certificate) error
+}
+
+type pgStore struct {
+	*sqlx.DB
+}
+
+func NewDB(driver, conn string, logger kitlog.Logger) (Datastore, error) {
+	switch driver {
+	case "postgres":
+		db, err := sqlx.Open(driver, conn)
+		if err != nil {
+			return nil, errors.Wrap(err, "certificates datastore")
+		}
+		var dbError error
+		maxAttempts := 20
+		for attempts := 1; attempts <= maxAttempts; attempts++ {
+			dbError = db.Ping()
+			if dbError == nil {
+				break
+			}
+			logger.Log("msg", fmt.Sprintf("could not connect to postgres: %v", dbError))
+			time.Sleep(time.Duration(attempts) * time.Second)
+		}
+		if dbError != nil {
+			return nil, errors.Wrap(dbError, "applications datastore")
+		}
+		return pgStore{DB: db}, nil
+	default:
+		return nil, errors.New("unknown driver")
+	}
+}
+
+func (store pgStore) New(c *Certificate) (string, error) {
+	if err := store.QueryRow(insertCertificateStmt, c.DeviceUUID, c.CommonName, "", c.IsIdentity).Scan(&c.UUID); err != nil {
+		return "", err
+	}
+
+	return c.UUID, nil
+}
+
+func (store pgStore) Certificates(params ...interface{}) ([]Certificate, error) {
+	stmt := selectCertificatesStmt
+	stmt = addWhereFilters(stmt, "OR", params...)
+	var certificates []Certificate
+	err := store.Select(&certificates, stmt)
+	if err != nil {
+		return nil, errors.Wrap(err, "pgStore Certificates")
+	}
+	return certificates, nil
+}
+
+func (store pgStore) GetCertificatesByDeviceUDID(udid string) ([]Certificate, error) {
+	var certificates []Certificate
+	err := store.Select(&certificates, selectCertificatesByDeviceUdidStmt, udid)
+	if err != nil {
+		return nil, errors.Wrap(err, "pgStore GetCertificatesByDeviceUDID")
+	}
+	return certificates, nil
+}
+
+func (store pgStore) GetCertificatesByDeviceUUID(uuid string) ([]Certificate, error) {
+	var certificates []Certificate
+	err := store.Select(&certificates, selectCertificatesByDeviceUuidStmt, uuid)
+	if err != nil {
+		return nil, errors.Wrap(err, "pgStore GetCertificatesByDeviceUUID")
+	}
+	return certificates, nil
+}
+
+func (store pgStore) ReplaceCertificatesByDeviceUUID(uuid string, certificates []Certificate) error {
+	tx, err := store.Beginx()
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+
+	tx.MustExec("DELETE FROM devices_certificates WHERE device_uuid = $1", uuid)
+
+	var insertedUuids []string = []string{}
+	for _, cert := range certificates {
+		if err := tx.QueryRow(insertCertificateStmt, cert.DeviceUUID, cert.CommonName, "", cert.IsIdentity).Scan(&cert.UUID); err != nil {
+			tx.Rollback()
+			return err
+		}
+
+		insertedUuids = append(insertedUuids, cert.UUID)
+	}
+
+	tx.Commit()
+	fmt.Println(strings.Join(insertedUuids, ","))
+	return nil
+}
+
+// add WHERE clause from params
+func addWhereFilters(stmt string, separator string, params ...interface{}) string {
+	var where []string
+	for _, param := range params {
+		if f, ok := param.(whereer); ok {
+			where = append(where, f.where())
+		}
+	}
+
+	if len(where) != 0 {
+		whereFilter := strings.Join(where, " "+separator+" ")
+		stmt = fmt.Sprintf("%s WHERE %s", stmt, whereFilter)
+	}
+	return stmt
+}

certificates/datastore_test.go

@@ -0,0 +1,41 @@
+package certificates
+
+import (
+	"database/sql"
+	"github.com/jmoiron/sqlx"
+	"gopkg.in/DATA-DOG/go-sqlmock.v1"
+	"testing"
+)
+
+var (
+	db   *sql.DB
+	dbx  *sqlx.DB
+	mock sqlmock.Sqlmock
+	err  error
+)
+
+func setup() {
+	db, mock, err = sqlmock.New()
+	if err != nil {
+		panic("an error was not expected when opening a stub database connection")
+	}
+	dbx = sqlx.NewDb(db, "mock")
+}
+
+func teardown() {
+	dbx.Close()
+}
+
+func NewDatastore(connection *sqlx.DB) (Datastore, error) {
+	return pgStore{DB: connection}, nil
+}
+
+func TestNewDatastore(t *testing.T) {
+	setup()
+	defer teardown()
+
+	_, err := NewDatastore(dbx)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

connect/endpoint.go

@@ -55,6 +55,20 @@ func makeConnectEndpoint(svc Service) endpoint.Endpoint {
 				return mdmConnectResponse{}, nil
 			}
 			return mdmConnectResponse{payload: next}, nil
+		case "Error":
+			total, err := svc.FailCommand(ctx, req.Response)
+			if err != nil {
+				return mdmConnectResponse{Err: err}, nil
+			}
+			// TODO: Deal with command failures
+			if total != 0 {
+				next, _, err := svc.NextCommand(ctx, req.Response)
+				if err != nil {
+					return mdmConnectResponse{Err: err}, nil
+				}
+				return mdmConnectResponse{payload: next}, nil
+			}
+
 		default:
 			return mdmConnectResponse{Err: errInvalidMessageType}, nil
 		}

connect/service.go

@@ -22,11 +22,12 @@ type Service interface {
 }
 
 // NewService creates a mdm service
-func NewService(devices device.Datastore, apps apps.Datastore, cs command.Service) Service {
+func NewService(devices device.Datastore, apps apps.Datastore, certs certificates.Datastore, cs command.Service) Service {
 	return &service{
 		commands: cs,
 		devices:  devices,
 		apps:     apps,
+		certs:    certs,
 	}
 }
 
@@ -34,6 +35,7 @@ type service struct {
 	devices  device.Datastore
 	apps     apps.Datastore
 	commands command.Service
+	certs    certificates.Datastore
 }
 
 // Acknowledge a response from a device.

enroll/service.go

@@ -73,6 +73,7 @@ func (svc service) Enroll(ctx context.Context) (Profile, error) {
 	scepPayload.PayloadDescription = "Configures SCEP"
 	scepPayload.PayloadDisplayName = "SCEP"
 	scepPayload.PayloadIdentifier = "com.github.micromdm.scep"
+	scepPayload.PayloadOrganization = "MicroMDM"
 	scepPayload.PayloadContent = scepContent
 	scepPayload.PayloadScope = "System"
 

main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/go-kit/kit/log"
 	"github.com/micromdm/dep"
 	"github.com/micromdm/micromdm/applications"
+	"github.com/micromdm/micromdm/certificates"
 	"github.com/micromdm/micromdm/checkin"
 	"github.com/micromdm/micromdm/command"
 	"github.com/micromdm/micromdm/connect"
@@ -203,15 +204,12 @@ func main() {
 	commandSvc := command.NewService(commandDB)
 	checkinSvc := checkin.NewService(deviceDB, mgmtSvc, commandSvc, enrollmentProfile)
 	connectSvc := connect.NewService(deviceDB, commandSvc)
-	connectSvc := connect.NewService(deviceDB, appsDB, commandSvc)
-	enrollSvc, _ := enroll.NewService(*flPushCert, *flPushPass, *flTLSCACert, *flSCEPURL, *flURL)
 
 	httpLogger := log.NewContext(logger).With("component", "http")
 	managementHandler := management.ServiceHandler(ctx, mgmtSvc, httpLogger)
 	commandHandler := command.ServiceHandler(ctx, commandSvc, httpLogger)
 	checkinHandler := checkin.ServiceHandler(ctx, checkinSvc, httpLogger)
 	connectHandler := connect.ServiceHandler(ctx, connectSvc, httpLogger)
-	enrollHandler := enroll.ServiceHandler(ctx, enrollSvc, httpLogger)
 
 	mux := http.NewServeMux()
 

management/endpoint_certificates.go

@@ -0,0 +1,41 @@
+package management
+
+import (
+	"encoding/json"
+	"github.com/go-kit/kit/endpoint"
+	"github.com/micromdm/micromdm/certificates"
+	"golang.org/x/net/context"
+	"net/http"
+)
+
+type listCertificatesRequest struct {
+	UUID string
+}
+
+type listCertificatesResponse struct {
+	certificates []certificates.Certificate `json:"certificates,omitempty"`
+	Err          error                      `json:"error,omitempty"`
+}
+
+func (r listCertificatesResponse) error() error { return r.Err }
+
+func (r listCertificatesResponse) encodeList(w http.ResponseWriter) error {
+	jsn, err := json.MarshalIndent(r.certificates, "", "  ")
+	if err != nil {
+		return err
+	}
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.Write(jsn)
+	return nil
+}
+
+func makeCertificatesEndpoint(svc Service) endpoint.Endpoint {
+	return func(ctx context.Context, request interface{}) (interface{}, error) {
+		req := request.(listCertificatesRequest)
+		certs, err := svc.Certificates(req.UUID)
+		if err != nil {
+			return listCertificatesResponse{Err: err}, nil
+		}
+		return listCertificatesResponse{certificates: certs}, nil
+	}
+}

management/service.go

@@ -5,6 +5,7 @@ import (
 	"github.com/RobotsAndPencils/buford/push"
 	"github.com/micromdm/dep"
 	"github.com/micromdm/micromdm/applications"
+	"github.com/micromdm/micromdm/certificates"
 	"github.com/micromdm/micromdm/device"
 	"github.com/micromdm/micromdm/workflow"
 	"github.com/pkg/errors"
@@ -31,6 +32,9 @@ type Service interface {
 	// Installed Applications
 	InstalledApps(deviceUUID string) ([]applications.Application, error)
 
+	// Installed Certificates
+	Certificates(deviceUUID string) ([]certificates.Certificate, error)
+
 	// AssignWorkflow assigns a workflow to a device
 	AssignWorkflow(deviceUUID, workflowUUID string) error
 
@@ -43,13 +47,14 @@ type Service interface {
 }
 
 // NewService creates a management service
-func NewService(ds device.Datastore, ws workflow.Datastore, dc dep.Client, ps *push.Service, as applications.Datastore) Service {
+func NewService(ds device.Datastore, ws workflow.Datastore, dc dep.Client, ps *push.Service, as applications.Datastore, cs certificates.Datastore) Service {
 	return &service{
 		devices:      ds,
 		depClient:    dc,
 		workflows:    ws,
 		pushsvc:      ps,
 		applications: as,
+		certificates: cs,
 	}
 }
 
@@ -59,6 +64,7 @@ type service struct {
 	workflows    workflow.Datastore
 	pushsvc      *push.Service
 	applications applications.Datastore
+	certificates certificates.Datastore
 }
 
 func (svc service) Push(deviceUDID string) (string, error) {
@@ -173,3 +179,12 @@ func (svc service) InstalledApps(deviceUUID string) ([]applications.Application,
 
 	return apps, nil
 }
+
+func (svc service) Certificates(deviceUUID string) ([]certificates.Certificate, error) {
+	certs, err := svc.certificates.GetCertificatesByDeviceUUID(deviceUUID)
+	if err != nil {
+		return nil, errors.Wrap(err, "management: certificates")
+	}
+
+	return certs, nil
+}

management/service_test.go

@@ -0,0 +1,23 @@
+package management
+
+import "testing"
+
+func svcSetup() {
+
+}
+
+func svcTearDown() {
+
+}
+
+func TestService_InstalledApps(t *testing.T) {
+	svcSetup()
+	defer svcTearDown()
+
+	svc := NewService(nil, nil, nil, nil, nil)
+	_, err := svc.InstalledApps("00000000-1111-2222-3333-444455556666")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+}

management/transport.go

@@ -108,6 +108,13 @@ func ServiceHandler(ctx context.Context, svc Service, logger kitlog.Logger) http
 		encodeResponse,
 		opts...,
 	)
+	certificatesHandler := kithttp.NewServer(
+		ctx,
+		makeCertificatesEndpoint(svc),
+		decodeCertificatesRequest,
+		encodeResponse,
+		opts...,
+	)
 
 	r := mux.NewRouter()
 
@@ -119,6 +126,7 @@ func ServiceHandler(ctx context.Context, svc Service, logger kitlog.Logger) http
 	r.Handle("/management/v1/devices/{uuid}", updateDeviceHandler).Methods("PATCH")
 	r.Handle("/management/v1/devices/{udid}/push", pushHandler).Methods("POST")
 	r.Handle("/management/v1/devices/{uuid}/applications", installedAppsHandler).Methods("GET")
+	r.Handle("/management/v1/devices/{uuid}/certificates", certificatesHandler).Methods("GET")
 	// profiles
 	r.Handle("/management/v1/profiles", addProfileHandler).Methods("POST")
 	r.Handle("/management/v1/profiles", listProfilesHandler).Methods("GET")
@@ -246,6 +254,16 @@ func decodeInstalledAppsRequest(_ context.Context, r *http.Request) (interface{}
 	return request, nil
 }
 
+func decodeCertificatesRequest(_ context.Context, r *http.Request) (interface{}, error) {
+	vars := mux.Vars(r)
+	uuid, ok := vars["uuid"]
+	if !ok {
+		return nil, errBadRouting
+	}
+
+	return listCertificatesRequest{UUID: uuid}, nil
+}
+
 func encodeResponse(ctx context.Context, w http.ResponseWriter, response interface{}) error {
 	if e, ok := response.(errorer); ok && e.error() != nil {
 		encodeError(ctx, e.error(), w)

migrations/201607110002_devices_certificates_down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS devices_certificates;

migrations/201607110002_devices_certificates_up.sql

@@ -0,0 +1,10 @@
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+CREATE TABLE IF NOT EXISTS devices_certificates (
+  certificate_uuid uuid PRIMARY KEY DEFAULT uuid_generate_v4(),
+  device_uuid uuid REFERENCES devices(device_uuid) ON DELETE CASCADE,
+  common_name text NOT NULL,
+  data BYTEA NOT NULL,
+  is_identity BOOL DEFAULT false
+)
+