From 17bce22e9507a5dc6f0f9e4ce017011cffc02504 Mon Sep 17 00:00:00 2001 From: Victor Vrantchan Date: Sat, 13 Oct 2018 21:05:19 -0400 Subject: [PATCH] add postgres device store (#531) --- .circleci/config.yml | 10 ++ Makefile | 36 +++++ docker-compose-dev.yaml | 11 ++ go.mod | 19 ++- go.sum | 36 ++++- pg/migrations/00001_tables.sql | 32 +++++ pkg/tools/tools.go | 7 + platform/device/builtin/db.go | 13 +- platform/device/builtin/db_test.go | 23 ++-- platform/device/device.go | 61 ++++----- platform/device/get_devices.go | 2 +- platform/device/pg/pg.go | 212 +++++++++++++++++++++++++++++ platform/device/pg/pg_test.go | 73 ++++++++++ platform/device/remove_devices.go | 4 +- platform/device/service.go | 6 +- platform/device/worker.go | 36 ++--- 16 files changed, 500 insertions(+), 81 deletions(-) create mode 100644 docker-compose-dev.yaml create mode 100644 pg/migrations/00001_tables.sql create mode 100644 pkg/tools/tools.go create mode 100644 platform/device/pg/pg.go create mode 100644 platform/device/pg/pg_test.go diff --git a/.circleci/config.yml b/.circleci/config.yml index f7e6ba24..a09bd3a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,10 +3,20 @@ jobs: build-go1.11: docker: - image: golang:1.11 + - image: postgres:9.6 + environment: + POSTGRES_DB: micromdm + POSTGRES_USER: micromdm + POSTGRES_PASSWORD: micromdm + working_directory: /go/src/github.com/micromdm/micromdm steps: &steps - checkout + - run: apt-get update -y && apt-get install postgresql-client -y - run: make deps + - run: GO111MODULE=on go install github.com/pressly/goose/cmd/goose + - run: make db-reset-test + - run: make db-migrate-test - run: make test - run: make diff --git a/Makefile b/Makefile index 4b1dd3b4..1728905c 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,10 @@ else CURRENT_PLATFORM = windows endif +ifeq ($(PG_HOST),) +PG_HOST := localhost +endif + BUILD_VERSION = "\ -X github.com/micromdm/go4/version.appName=${APP_NAME} \ -X github.com/micromdm/go4/version.version=${VERSION} \ @@ -107,3 +111,35 @@ docker-tag: docker-build ngrok: @./tools/ngrok/screen + +docker-compose: + docker-compose -f docker-compose-dev.yaml up -d + +db-psql-test: + $(call psql_db,micromdm_test) + +db-psql: + $(call psql_db,micromdm) + +define psql_db + PGPASSWORD=micromdm psql --host=${PG_HOST} --port=5432 --username=micromdm --dbname=$(1) +endef + +db-reset-test: + $(call psql_exec,'DROP DATABASE IF EXISTS micromdm_test;') + $(call psql_exec,'CREATE DATABASE micromdm_test;') + +define psql_exec + PGPASSWORD=micromdm psql --host=${PG_HOST} --port=5432 --username=micromdm -c $(1) +endef + +db-migrate-test: + $(call goose_up,micromdm_test) + +db-migrate: + $(call goose_up,micromdm) + +define goose_up + cd ./pg/migrations && goose postgres "host=${PG_HOST} port=5432 user=micromdm dbname=$(1) password=micromdm sslmode=disable" up +endef + diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml new file mode 100644 index 00000000..74067925 --- /dev/null +++ b/docker-compose-dev.yaml @@ -0,0 +1,11 @@ +version: "2" + +services: + postgresql: + image: postgres:9.6 + environment: + POSTGRES_DB: micromdm + POSTGRES_USER: micromdm + POSTGRES_PASSWORD: micromdm + ports: + - 5432:5432 diff --git a/go.mod b/go.mod index 672456d4..10fc07e9 100644 --- a/go.mod +++ b/go.mod @@ -3,28 +3,41 @@ module github.com/micromdm/micromdm require ( github.com/RobotsAndPencils/buford v0.12.0 github.com/boltdb/bolt v1.3.1 + github.com/davecgh/go-spew v1.1.1 // indirect github.com/fullsailor/pkcs7 v0.0.0-20180824154052-36585635cb64 github.com/garyburd/go-oauth v0.0.0-20180319155456-bca2e7f09a17 github.com/go-kit/kit v0.7.0 github.com/go-logfmt/logfmt v0.3.0 // indirect + github.com/go-sql-driver/mysql v1.4.0 // indirect github.com/go-stack/stack v1.7.0 // indirect github.com/gogo/protobuf v1.0.0 - github.com/golang/protobuf v1.1.0 + github.com/golang/protobuf v1.2.0 github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/mux v1.6.2 github.com/groob/finalizer v0.0.0-20170707115354-4c2ed49aabda github.com/groob/plist v0.0.0-20180203051248-dd56909aee38 + github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 + github.com/kolide/kit v0.0.0-20180912215818-0c28f72eb2b0 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 // indirect github.com/kr/pretty v0.1.0 // indirect + github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect + github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/lib/pq v1.0.0 + github.com/mattn/go-sqlite3 v1.9.0 // indirect github.com/micromdm/go4 v0.0.0-20171021081444-deded5397014 github.com/micromdm/scep v1.0.1-0.20180906231441-a136542b4bc9 github.com/pkg/errors v0.8.0 + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pressly/goose v2.3.0+incompatible github.com/satori/go.uuid v1.2.0 + github.com/stretchr/testify v1.2.2 // indirect + github.com/ziutek/mymysql v1.5.4 // indirect golang.org/x/crypto v0.0.0-20180614174826-fd5f17ee7299 - golang.org/x/net v0.0.0-20180611182652-db08ff08e862 + golang.org/x/net v0.0.0-20180724234803-3673e40ba225 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect golang.org/x/sys v0.0.0-20180614134839-8883426083c0 // indirect - golang.org/x/text v0.3.0 // indirect + google.golang.org/appengine v1.2.0 // indirect + gopkg.in/Masterminds/squirrel.v1 v1.0.0-20170825200431-a6b93000bd21 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index f94719ad..5fd2af20 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/RobotsAndPencils/buford v0.12.0 h1:2nfOk+N/QVoQHwXIS0m5TFdvlUjEnqAj/0 github.com/RobotsAndPencils/buford v0.12.0/go.mod h1:27KhJZ/wLQHRnsZF+mTWKvF5w8U4dVl4Nh+BfQem4Lo= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fullsailor/pkcs7 v0.0.0-20180824154052-36585635cb64 h1:gCCjmcHGY24tPnwyj/AsUBzZFdZljZW1WaBNAmHRdw0= github.com/fullsailor/pkcs7 v0.0.0-20180824154052-36585635cb64/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/garyburd/go-oauth v0.0.0-20180319155456-bca2e7f09a17 h1:GOfMz6cRgTJ9jWV0qAezv642OhPnKEG7gtUjJSdStHE= @@ -10,12 +12,14 @@ github.com/go-kit/kit v0.7.0 h1:ApufNmWF1H6/wUbAG81hZOHmqwd0zRf8mNfLjYj/064= github.com/go-kit/kit v0.7.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.7.0 h1:S04+lLfST9FvL8dl4R31wVUC/paZp/WQZbLmUgWboGw= github.com/go-stack/stack v1.7.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.0.0 h1:2jyBKDKU/8v3v2xVR2PtiWQviFUyiaGk2rpfyFT8rTM= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc= -github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= @@ -26,6 +30,10 @@ github.com/groob/pkcs7 v0.0.0-20180824154052-36585635cb64 h1:1ALD84dEnUxPKZENhUA github.com/groob/pkcs7 v0.0.0-20180824154052-36585635cb64/go.mod h1:mEOMQ8C7oeXY3LnE2jy4UkLAqrW9rrpwiP5U4hVV+MY= github.com/groob/plist v0.0.0-20180203051248-dd56909aee38 h1:afbUddvIjPRC7XHHgeSTRfzZtIxEsSl4VCxumLBGDJU= github.com/groob/plist v0.0.0-20180203051248-dd56909aee38/go.mod h1:qg2Nek0ND/hIr+nY8H1oVqEW2cLzVVNaAQ0QexOyjyc= +github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE= +github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= +github.com/kolide/kit v0.0.0-20180912215818-0c28f72eb2b0 h1:8UB0NJBoB/lN2O9zG9+U2klyYE0UhvN2Zrn7HBjhwNY= +github.com/kolide/kit v0.0.0-20180912215818-0c28f72eb2b0/go.mod h1:N3Yv8okDVC/5qZhPA9uxVYRfkp4mD2vrlQiSCWlNCpg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -33,23 +41,43 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= +github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/micromdm/go4 v0.0.0-20171021081444-deded5397014 h1:8Za9WLoGTSU96EAWFziRCiFmQuBAC6xj9/dKiN45LHw= github.com/micromdm/go4 v0.0.0-20171021081444-deded5397014/go.mod h1:8EzTEgA3q2ZdZotWXs1bWnFCXuaFHU0+jDNZbHlwduM= github.com/micromdm/scep v1.0.1-0.20180906231441-a136542b4bc9 h1:LGgQOrgBGOoye+MmS1YT1XpdlOEEkqc/1c0XBPBELnw= github.com/micromdm/scep v1.0.1-0.20180906231441-a136542b4bc9/go.mod h1:CID2SixSr5FvoauZdAFUSpQkn5MAuSy9oyURMGOJbag= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pressly/goose v2.3.0+incompatible h1:Nc9o+JsN4j8sS4hvRzcfKYOrr7W2EXMDY2wNYtKmaVc= +github.com/pressly/goose v2.3.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= golang.org/x/crypto v0.0.0-20180614174826-fd5f17ee7299 h1:zxP+xTjjk4kD+M5IFPweL7/4851FUhYkzbDqbzkN1JE= golang.org/x/crypto v0.0.0-20180614174826-fd5f17ee7299/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/net v0.0.0-20180611182652-db08ff08e862 h1:JZi6BqOZ+iSgmLWe6llhGrNnEnK+YB/MRkStwnEfbqM= -golang.org/x/net v0.0.0-20180611182652-db08ff08e862/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225 h1:kNX+jCowfMYzvlSvJu5pQWEmyWFrBXJ3PBy10xKMXK8= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180614134839-8883426083c0 h1:5mOaSPjCt3RW5w1KpSFOVg8VdqQQ/FjfM5/m50f/8wM= golang.org/x/sys v0.0.0-20180614134839-8883426083c0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/Masterminds/squirrel.v1 v1.0.0-20170825200431-a6b93000bd21 h1:GmGVIcDxdecAcVjcTp4IpK4VmCMxXhyZKwN2eIzsZ4Y= +gopkg.in/Masterminds/squirrel.v1 v1.0.0-20170825200431-a6b93000bd21/go.mod h1:8PH4rQjb7OdPC6OWDDuY6J/PT8iSNTiff3jmccc2m10= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pg/migrations/00001_tables.sql b/pg/migrations/00001_tables.sql new file mode 100644 index 00000000..bc1d68e5 --- /dev/null +++ b/pg/migrations/00001_tables.sql @@ -0,0 +1,32 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS devices ( + uuid TEXT PRIMARY KEY, + udid TEXT DEFAULT '', + serial_number TEXT DEFAULT '', + os_version TEXT DEFAULT '', + build_version TEXT DEFAULT '', + product_name TEXT DEFAULT '', + imei TEXT DEFAULT '', + meid TEXT DEFAULT '', + push_magic TEXT DEFAULT '', + awaiting_configuration BOOLEAN DEFAULT false, + token TEXT DEFAULT '', + unlock_token TEXT DEFAULT '', + enrolled BOOLEAN DEFAULT false, + description TEXT DEFAULT '', + model TEXT DEFAULT '', + model_name TEXT DEFAULT '', + device_name TEXT DEFAULT '', + color TEXT DEFAULT '', + asset_tag TEXT DEFAULT '', + dep_profile_status TEXT DEFAULT '', + dep_profile_uuid TEXT DEFAULT '', + dep_profile_assign_time TIMESTAMP DEFAULT '1970-01-01 00:00:00', + dep_profile_push_time TIMESTAMP DEFAULT '1970-01-01 00:00:00', + dep_profile_assigned_date TIMESTAMP DEFAULT '1970-01-01 00:00:00', + dep_profile_assigned_by TEXT DEFAULT '', + last_seen TIMESTAMP DEFAULT '1970-01-01 00:00:00' +); + +-- +goose Down +DROP TABLE IF EXISTS devices; diff --git a/pkg/tools/tools.go b/pkg/tools/tools.go new file mode 100644 index 00000000..d311e6e8 --- /dev/null +++ b/pkg/tools/tools.go @@ -0,0 +1,7 @@ +// +build tools + +package tools + +import ( + _ "github.com/pressly/goose/cmd/goose" +) diff --git a/platform/device/builtin/db.go b/platform/device/builtin/db.go index 02fabc5c..9710c065 100644 --- a/platform/device/builtin/db.go +++ b/platform/device/builtin/db.go @@ -1,6 +1,7 @@ package builtin import ( + "context" "fmt" "github.com/boltdb/bolt" @@ -45,7 +46,7 @@ func NewDB(db *bolt.DB) (*DB, error) { return datastore, nil } -func (db *DB) List(opt device.ListDevicesOption) ([]device.Device, error) { +func (db *DB) List(ctx context.Context, opt device.ListDevicesOption) ([]device.Device, error) { var devices []device.Device err := db.View(func(tx *bolt.Tx) error { b := tx.Bucket([]byte(DeviceBucket)) @@ -71,7 +72,7 @@ func (db *DB) List(opt device.ListDevicesOption) ([]device.Device, error) { return devices, err } -func (db *DB) Save(dev *device.Device) error { +func (db *DB) Save(ctx context.Context, dev *device.Device) error { tx, err := db.DB.Begin(true) if err != nil { return errors.Wrap(err, "begin transaction") @@ -109,11 +110,11 @@ func (db *DB) Save(dev *device.Device) error { return tx.Commit() } -func (db *DB) DeleteByUDID(udid string) error { +func (db *DB) DeleteByUDID(ctx context.Context, udid string) error { return db.deleteByIndex(udid) } -func (db *DB) DeleteBySerial(serial string) error { +func (db *DB) DeleteBySerial(ctx context.Context, serial string) error { return db.deleteByIndex(serial) } @@ -157,11 +158,11 @@ func (e *notFound) NotFound() bool { return true } -func (db *DB) DeviceByUDID(udid string) (*device.Device, error) { +func (db *DB) DeviceByUDID(ctx context.Context, udid string) (*device.Device, error) { return db.deviceByIndex(udid) } -func (db *DB) DeviceBySerial(serial string) (*device.Device, error) { +func (db *DB) DeviceBySerial(ctx context.Context, serial string) (*device.Device, error) { return db.deviceByIndex(serial) } diff --git a/platform/device/builtin/db_test.go b/platform/device/builtin/db_test.go index fcbf5bdc..8b024580 100644 --- a/platform/device/builtin/db_test.go +++ b/platform/device/builtin/db_test.go @@ -1,6 +1,7 @@ package builtin import ( + "context" "io/ioutil" "os" "testing" @@ -18,17 +19,18 @@ func TestSave(t *testing.T) { SerialNumber: "foobarbaz", ProductName: "MacBook", } + ctx := context.Background() - if err := db.Save(dev); err != nil { + if err := db.Save(ctx, dev); err != nil { t.Fatalf("saving device in datastore: %s", err) } - byUDID, err := db.DeviceByUDID(dev.UDID) + byUDID, err := db.DeviceByUDID(ctx, dev.UDID) if err != nil { t.Fatalf("getting device by UDID: %s", err) } - bySerial, err := db.DeviceBySerial(dev.SerialNumber) + bySerial, err := db.DeviceBySerial(ctx, dev.SerialNumber) if err != nil { t.Fatalf("getting device by UDID: %s", err) } @@ -72,16 +74,17 @@ func TestDeleteByUDID(t *testing.T) { SerialNumber: "foobarbaz", ProductName: "MacBook", } + ctx := context.Background() - if err := db.Save(dev); err != nil { + if err := db.Save(ctx, dev); err != nil { t.Fatalf("saving device in datastore: %s", err) } - if err := db.DeleteByUDID(dev.UDID); err != nil { + if err := db.DeleteByUDID(ctx, dev.UDID); err != nil { t.Fatalf("deleting device in datastore: %s", err) } - byUDID, _ := db.DeviceByUDID(dev.UDID) + byUDID, _ := db.DeviceByUDID(ctx, dev.UDID) if byUDID != nil { t.Fatalf("expected device to be deleted") } @@ -96,15 +99,17 @@ func TestDeleteBySerial(t *testing.T) { ProductName: "MacBook", } - if err := db.Save(dev); err != nil { + ctx := context.Background() + + if err := db.Save(ctx, dev); err != nil { t.Fatalf("saving device in datastore: %s", err) } - if err := db.DeleteBySerial(dev.SerialNumber); err != nil { + if err := db.DeleteBySerial(ctx, dev.SerialNumber); err != nil { t.Fatalf("deleting device in datastore: %s", err) } - byUDID, _ := db.DeviceBySerial(dev.SerialNumber) + byUDID, _ := db.DeviceBySerial(ctx, dev.SerialNumber) if byUDID != nil { t.Fatalf("expected device to be deleted") } diff --git a/platform/device/device.go b/platform/device/device.go index c74b4b86..562413fc 100644 --- a/platform/device/device.go +++ b/platform/device/device.go @@ -12,35 +12,32 @@ import ( const DeviceEnrolledTopic = "mdm.DeviceEnrolled" type Device struct { - UUID string - UDID string - SerialNumber string - OSVersion string - BuildVersion string - ProductName string - IMEI string - MEID string - MDMTopic string - PushMagic string - AwaitingConfiguration bool - Token string - UnlockToken string - Enrolled bool - DEPDevice bool - Description string - Model string - ModelName string - DeviceName string - Color string - AssetTag string - DEPProfileStatus DEPProfileStatus - DEPProfileUUID string - DEPProfileAssignTime time.Time - DEPProfilePushTime time.Time - DEPProfileAssignedDate time.Time - DEPProfileAssignedBy string - LastSeen time.Time - LastQueryResponse []byte + UUID string `db:"uuid"` + UDID string `db:"udid"` + SerialNumber string `db:"serial_number"` + OSVersion string `db:"os_version"` + BuildVersion string `db:"build_version"` + ProductName string `db:"product_name"` + IMEI string `db:"imei"` + MEID string `db:"meid"` + PushMagic string `db:"push_magic"` + AwaitingConfiguration bool `db:"awaiting_configuration"` + Token string `db:"token"` + UnlockToken string `db:"unlock_token"` + Enrolled bool `db:"enrolled"` + Description string `db:"description"` + Model string `db:"model"` + ModelName string `db:"model_name"` + DeviceName string `db:"device_name"` + Color string `db:"color"` + AssetTag string `db:"asset_tag"` + DEPProfileStatus DEPProfileStatus `db:"dep_profile_status"` + DEPProfileUUID string `db:"dep_profile_uuid"` + DEPProfileAssignTime time.Time `db:"dep_profile_assign_time"` + DEPProfilePushTime time.Time `db:"dep_profile_push_time"` + DEPProfileAssignedDate time.Time `db:"dep_profile_assigned_date"` + DEPProfileAssignedBy string `db:"dep_profile_assigned_by"` + LastSeen time.Time `db:"last_seen"` } // DEPProfileStatus is the status of the DEP Profile @@ -67,7 +64,6 @@ func MarshalDevice(dev *Device) ([]byte, error) { Meid: dev.MEID, Token: dev.Token, PushMagic: dev.PushMagic, - MdmTopic: dev.MDMTopic, UnlockToken: dev.UnlockToken, Enrolled: dev.Enrolled, AwaitingConfiguration: dev.AwaitingConfiguration, @@ -77,7 +73,6 @@ func MarshalDevice(dev *Device) ([]byte, error) { Description: dev.Description, Color: dev.Color, AssetTag: dev.AssetTag, - DepDevice: dev.DEPDevice, DepProfileStatus: string(dev.DEPProfileStatus), DepProfileUuid: dev.DEPProfileUUID, DepProfileAssignTime: timeToNano(dev.DEPProfileAssignTime), @@ -85,7 +80,6 @@ func MarshalDevice(dev *Device) ([]byte, error) { DepProfileAssignedDate: timeToNano(dev.DEPProfileAssignedDate), DepProfileAssignedBy: dev.DEPProfileAssignedBy, LastSeen: timeToNano(dev.LastSeen), - LastQueryResponse: dev.LastQueryResponse, } return proto.Marshal(&protodev) } @@ -105,7 +99,6 @@ func UnmarshalDevice(data []byte, dev *Device) error { dev.MEID = pb.GetMeid() dev.Token = pb.GetToken() dev.PushMagic = pb.GetPushMagic() - dev.MDMTopic = pb.GetMdmTopic() dev.UnlockToken = pb.GetUnlockToken() dev.Enrolled = pb.GetEnrolled() dev.AwaitingConfiguration = pb.GetAwaitingConfiguration() @@ -115,7 +108,6 @@ func UnmarshalDevice(data []byte, dev *Device) error { dev.Description = pb.GetDescription() dev.Color = pb.GetColor() dev.AssetTag = pb.GetAssetTag() - dev.DEPDevice = pb.GetDepDevice() dev.DEPProfileStatus = DEPProfileStatus(pb.GetDepProfileStatus()) dev.DEPProfileUUID = pb.GetDepProfileUuid() dev.DEPProfileAssignTime = timeFromNano(pb.GetDepProfileAssignTime()) @@ -123,7 +115,6 @@ func UnmarshalDevice(data []byte, dev *Device) error { dev.DEPProfileAssignedDate = timeFromNano(pb.GetDepProfileAssignedDate()) dev.DEPProfileAssignedBy = pb.GetDepProfileAssignedBy() dev.LastSeen = timeFromNano(pb.GetLastSeen()) - dev.LastQueryResponse = pb.GetLastQueryResponse() return nil } diff --git a/platform/device/get_devices.go b/platform/device/get_devices.go index d03b64e1..f041a085 100644 --- a/platform/device/get_devices.go +++ b/platform/device/get_devices.go @@ -26,7 +26,7 @@ type DeviceDTO struct { } func (svc *DeviceService) ListDevices(ctx context.Context, opt ListDevicesOption) ([]DeviceDTO, error) { - devices, err := svc.store.List(opt) + devices, err := svc.store.List(ctx, opt) var dto []DeviceDTO for _, d := range devices { dto = append(dto, DeviceDTO{ diff --git a/platform/device/pg/pg.go b/platform/device/pg/pg.go new file mode 100644 index 00000000..016aa448 --- /dev/null +++ b/platform/device/pg/pg.go @@ -0,0 +1,212 @@ +package pg + +import ( + "context" + "database/sql" + "strings" + + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" + sq "gopkg.in/Masterminds/squirrel.v1" + + "github.com/micromdm/micromdm/platform/device" +) + +type Postgres struct{ db *sqlx.DB } + +func New(db *sqlx.DB) *Postgres { + return &Postgres{db: db} +} + +func columns() []string { + return []string{ + "uuid", + "udid", + "serial_number", + "os_version", + "build_version", + "product_name", + "imei", + "meid", + "push_magic", + "awaiting_configuration", + "token", + "unlock_token", + "enrolled", + "description", + "model", + "model_name", + "device_name", + "color", + "asset_tag", + "dep_profile_status", + "dep_profile_uuid", + "dep_profile_assign_time", + "dep_profile_push_time", + "dep_profile_assigned_date", + "dep_profile_assigned_by", + "last_seen", + } +} + +const tableName = "devices" + +func (d *Postgres) Save(ctx context.Context, device *device.Device) error { + updateQuery, _, err := sq.StatementBuilder.PlaceholderFormat(sq.Dollar). + Update(tableName). + Prefix("ON CONFLICT (uuid) DO"). + Set("uuid", device.UUID). + Set("udid", device.UDID). + Set("serial_number", device.SerialNumber). + Set("os_version", device.OSVersion). + Set("build_version", device.BuildVersion). + Set("product_name", device.ProductName). + Set("imei", device.IMEI). + Set("meid", device.MEID). + Set("push_magic", device.PushMagic). + Set("awaiting_configuration", device.AwaitingConfiguration). + Set("token", device.Token). + Set("unlock_token", device.UnlockToken). + Set("enrolled", device.Enrolled). + Set("description", device.Description). + Set("model", device.Model). + Set("model_name", device.ModelName). + Set("device_name", device.DeviceName). + Set("color", device.Color). + Set("asset_tag", device.AssetTag). + Set("dep_profile_status", device.DEPProfileStatus). + Set("dep_profile_uuid", device.DEPProfileUUID). + Set("dep_profile_assign_time", device.DEPProfileAssignTime). + Set("dep_profile_push_time", device.DEPProfilePushTime). + Set("dep_profile_assigned_date", device.DEPProfileAssignedDate). + Set("dep_profile_assigned_by", device.DEPProfileAssignedBy). + Set("last_seen", device.LastSeen). + ToSql() + if err != nil { + return errors.Wrap(err, "building update query for device save") + } + updateQuery = strings.Replace(updateQuery, tableName, "", -1) + + query, args, err := sq.StatementBuilder.PlaceholderFormat(sq.Dollar). + Insert(tableName). + Columns(columns()...). + Values( + device.UUID, + device.UDID, + device.SerialNumber, + device.OSVersion, + device.BuildVersion, + device.ProductName, + device.IMEI, + device.MEID, + device.PushMagic, + device.AwaitingConfiguration, + device.Token, + device.UnlockToken, + device.Enrolled, + device.Description, + device.Model, + device.ModelName, + device.DeviceName, + device.Color, + device.AssetTag, + device.DEPProfileStatus, + device.DEPProfileUUID, + device.DEPProfileAssignTime, + device.DEPProfilePushTime, + device.DEPProfileAssignedDate, + device.DEPProfileAssignedBy, + device.LastSeen, + ). + Suffix(updateQuery). + ToSql() + if err != nil { + return errors.Wrap(err, "building device save query") + } + + _, err = d.db.ExecContext(ctx, query, args...) + return errors.Wrap(err, "exec device save in pg") +} + +func (d *Postgres) DeviceByUDID(ctx context.Context, udid string) (*device.Device, error) { + query, args, err := sq.StatementBuilder.PlaceholderFormat(sq.Dollar). + Select(columns()...). + From(tableName). + Where(sq.Eq{"udid": udid}). + ToSql() + if err != nil { + return nil, errors.Wrap(err, "building sql") + } + + var dev device.Device + err = d.db.QueryRowxContext(ctx, query, args...).StructScan(&dev) + if errors.Cause(err) == sql.ErrNoRows { + return nil, deviceNotFoundErr{} + } + return &dev, errors.Wrap(err, "finding device by udid") +} + +func (d *Postgres) DeviceBySerial(ctx context.Context, serial string) (*device.Device, error) { + query, args, err := sq.StatementBuilder.PlaceholderFormat(sq.Dollar). + Select(columns()...). + From(tableName). + Where(sq.Eq{"serial_number": serial}). + ToSql() + if err != nil { + return nil, errors.Wrap(err, "building sql") + } + + var dev device.Device + err = d.db.QueryRowxContext(ctx, query, args...).StructScan(&dev) + if errors.Cause(err) == sql.ErrNoRows { + return nil, deviceNotFoundErr{} + } + return &dev, errors.Wrap(err, "finding device by serial") +} + +func (d *Postgres) ListDevices(ctx context.Context, opt device.ListDevicesOption) ([]device.Device, error) { + query, args, err := sq.StatementBuilder.PlaceholderFormat(sq.Dollar). + Select(columns()...). + From(tableName). + ToSql() + if err != nil { + return nil, errors.Wrap(err, "building sql") + } + var list []device.Device + err = d.db.SelectContext(ctx, &list, query, args...) + return list, errors.Wrap(err, "list devices") +} + +func (d *Postgres) DeleteByUDID(ctx context.Context, udid string) error { + query, args, err := sq.StatementBuilder.PlaceholderFormat(sq.Dollar). + Delete(tableName). + Where(sq.Eq{"udid": udid}). + ToSql() + if err != nil { + return errors.Wrap(err, "building sql") + } + _, err = d.db.ExecContext(ctx, query, args...) + return errors.Wrap(err, "delete device by udid") +} + +func (d *Postgres) DeleteBySerial(ctx context.Context, serial string) error { + query, args, err := sq.StatementBuilder.PlaceholderFormat(sq.Dollar). + Delete(tableName). + Where(sq.Eq{"serial_number": serial}). + ToSql() + if err != nil { + return errors.Wrap(err, "building sql") + } + _, err = d.db.ExecContext(ctx, query, args...) + return errors.Wrap(err, "delete device by serial_number") +} + +type deviceNotFoundErr struct{} + +func (e deviceNotFoundErr) Error() string { + return "device not found" +} + +func (e deviceNotFoundErr) NotFound() bool { + return true +} diff --git a/platform/device/pg/pg_test.go b/platform/device/pg/pg_test.go new file mode 100644 index 00000000..aa45f41b --- /dev/null +++ b/platform/device/pg/pg_test.go @@ -0,0 +1,73 @@ +package pg + +import ( + "context" + "testing" + "time" + + "github.com/go-kit/kit/log" + "github.com/kolide/kit/dbutil" + _ "github.com/lib/pq" + "github.com/micromdm/micromdm/platform/device" +) + +func TestPGCrud(t *testing.T) { + db := setup(t) + ctx := context.Background() + + // create + dev := &device.Device{ + UUID: "foobar", + UDID: "foobar", + DEPProfileStatus: device.ASSIGNED, + LastSeen: time.Now().UTC(), + } + err := db.Save(ctx, dev) + if err != nil { + t.Fatal(err) + } + + // update + dev.DEPProfileStatus = device.PUSHED + err = db.Save(ctx, dev) + if err != nil { + t.Fatal(err) + } + + // find + found, err := db.DeviceByUDID(ctx, dev.UDID) + if err != nil { + t.Fatal(err) + } + + if have, want := found.DEPProfileStatus, dev.DEPProfileStatus; have != want { + t.Errorf("have %v, want %v", have, want) + } + + // list + devices, err := db.ListDevices(ctx, device.ListDevicesOption{}) + if err != nil { + t.Fatal(err) + } + + // delete + for _, dev := range devices { + if err := db.DeleteByUDID(ctx, dev.UDID); err != nil { + t.Fatal(err) + } + } +} + +func setup(t *testing.T) *Postgres { + db, err := dbutil.OpenDBX( + "postgres", + "host=localhost port=5432 user=micromdm dbname=micromdm_test password=micromdm sslmode=disable", + dbutil.WithLogger(log.NewNopLogger()), + dbutil.WithMaxAttempts(1), + ) + if err != nil { + t.Fatal(err) + } + + return New(db) +} diff --git a/platform/device/remove_devices.go b/platform/device/remove_devices.go index d0438910..eb471ad0 100644 --- a/platform/device/remove_devices.go +++ b/platform/device/remove_devices.go @@ -10,14 +10,14 @@ import ( func (svc *DeviceService) RemoveDevices(ctx context.Context, opt RemoveDevicesOptions) error { for _, udid := range opt.UDIDs { - err := svc.store.DeleteByUDID(udid) + err := svc.store.DeleteByUDID(ctx, udid) if err != nil { return err } } for _, serial := range opt.Serials { - err := svc.store.DeleteBySerial(serial) + err := svc.store.DeleteBySerial(ctx, serial) if err != nil { return err } diff --git a/platform/device/service.go b/platform/device/service.go index eb91177c..d71cf724 100644 --- a/platform/device/service.go +++ b/platform/device/service.go @@ -15,9 +15,9 @@ type Service interface { } type Store interface { - List(opt ListDevicesOption) ([]Device, error) - DeleteByUDID(udid string) error - DeleteBySerial(serial string) error + List(ctx context.Context, opt ListDevicesOption) ([]Device, error) + DeleteByUDID(ctx context.Context, udid string) error + DeleteBySerial(ctx context.Context, serial string) error } type DeviceService struct { diff --git a/platform/device/worker.go b/platform/device/worker.go index d14eb3f4..0c0c635e 100644 --- a/platform/device/worker.go +++ b/platform/device/worker.go @@ -15,9 +15,9 @@ import ( ) type DeviceWorkerStore interface { - Save(*Device) error - DeviceByUDID(udid string) (*Device, error) - DeviceBySerial(udid string) (*Device, error) + Save(ctx context.Context, d *Device) error + DeviceByUDID(ctx context.Context, udid string) (*Device, error) + DeviceBySerial(ctx context.Context, serial string) (*Device, error) } type Worker struct { @@ -94,7 +94,7 @@ func (w *Worker) updateFromDEPSync(ctx context.Context, message []byte) error { ) for _, dd := range ev.Devices { - dev, err := getOrCreateDeviceBySerial(w.db, dd.SerialNumber) + dev, err := getOrCreateDeviceBySerial(ctx, w.db, dd.SerialNumber) if err != nil { return errors.Wrap(err, "get device by serial") } @@ -127,7 +127,7 @@ func (w *Worker) updateFromDEPSync(ctx context.Context, message []byte) error { dev.DEPProfileAssignedDate = dd.DeviceAssignedDate dev.DEPProfileAssignedBy = dd.DeviceAssignedBy - if err := w.db.Save(dev); err != nil { + if err := w.db.Save(ctx, dev); err != nil { return errors.Wrap(err, "save device %s from DEP sync") } } @@ -141,13 +141,13 @@ func (w *Worker) updateFromAcknowledge(ctx context.Context, message []byte) erro return errors.Wrap(err, "unmarshal acknowledge event") } - dev, err := w.db.DeviceByUDID(ev.Response.UDID) + dev, err := w.db.DeviceByUDID(ctx, ev.Response.UDID) if err != nil { return errors.Wrapf(err, "retrieve device with udid %s", ev.Response.UDID) } dev.LastSeen = time.Now() - err = w.db.Save(dev) + err = w.db.Save(ctx, dev) return errors.Wrapf(err, "saving updated device for acknowledge event") } @@ -158,7 +158,7 @@ func (w *Worker) updateFromCheckout(ctx context.Context, message []byte) error { return errors.Wrap(err, "unmarshal checkin event") } - dev, err := w.db.DeviceByUDID(ev.Command.UDID) + dev, err := w.db.DeviceByUDID(ctx, ev.Command.UDID) if err != nil { return errors.Wrapf(err, "retrieve device with udid %s", ev.Command.UDID) } @@ -166,7 +166,7 @@ func (w *Worker) updateFromCheckout(ctx context.Context, message []byte) error { dev.Enrolled = false dev.LastSeen = time.Now() - err = w.db.Save(dev) + err = w.db.Save(ctx, dev) return errors.Wrapf(err, "saving updated device for checkout event") } @@ -182,7 +182,7 @@ func (w *Worker) updateFromTokenUpdate(ctx context.Context, message []byte) erro return nil } - dev, err := w.db.DeviceByUDID(ev.Command.UDID) + dev, err := w.db.DeviceByUDID(ctx, ev.Command.UDID) if err != nil { return errors.Wrapf(err, "retrieve device with udid %s", ev.Command.UDID) } @@ -194,7 +194,7 @@ func (w *Worker) updateFromTokenUpdate(ctx context.Context, message []byte) erro // first TokenUpdate event will have the enrollment status set to false. newlyEnrolled := !dev.Enrolled dev.Enrolled = true - if err := w.db.Save(dev); err != nil { + if err := w.db.Save(ctx, dev); err != nil { return errors.Wrapf(err, "saving updated device for Token event udid=%s", ev.Command.UDID) } @@ -213,7 +213,7 @@ func (w *Worker) updateFromAuthenticate(ctx context.Context, message []byte) err return errors.Wrap(err, "unmarshal checkin event") } - device, reenrolling, err := getOrCreateDevice(w.db, ev.Command.SerialNumber, ev.Command.UDID) + device, reenrolling, err := getOrCreateDevice(ctx, w.db, ev.Command.SerialNumber, ev.Command.UDID) if err != nil { return errors.Wrap(err, "get device for authenticate event") } @@ -244,16 +244,16 @@ func (w *Worker) updateFromAuthenticate(ctx context.Context, message []byte) err device.Model = ev.Command.Model device.ModelName = ev.Command.ModelName device.LastSeen = time.Now() - err = w.db.Save(device) + err = w.db.Save(ctx, device) return errors.Wrapf(err, "saving updated device for authenticate event") } -func getOrCreateDevice(db DeviceWorkerStore, serial, udid string) (dev *Device, reenrolling bool, err error) { +func getOrCreateDevice(ctx context.Context, db DeviceWorkerStore, serial, udid string) (dev *Device, reenrolling bool, err error) { if udid != "" { // first try to fetch a device by UDID. // If the device was previously enrolled it will exist. // In case the device is known, set the enrolled status to false before returning. - byUDID, err := db.DeviceByUDID(udid) + byUDID, err := db.DeviceByUDID(ctx, udid) if err == nil { byUDID.Enrolled = false return byUDID, true, nil @@ -265,12 +265,12 @@ func getOrCreateDevice(db DeviceWorkerStore, serial, udid string) (dev *Device, // next try to find the device by serial. If found, it's a DEP device, which contains only the // serials but not a udid. - dev, err = getOrCreateDeviceBySerial(db, serial) + dev, err = getOrCreateDeviceBySerial(ctx, db, serial) return dev, false, err } -func getOrCreateDeviceBySerial(db DeviceWorkerStore, serial string) (*Device, error) { - bySerial, err := db.DeviceBySerial(serial) +func getOrCreateDeviceBySerial(ctx context.Context, db DeviceWorkerStore, serial string) (*Device, error) { + bySerial, err := db.DeviceBySerial(ctx, serial) if err == nil && bySerial != nil { return bySerial, nil }