diff --git a/.gitignore b/.gitignore index c959f2f..101b28c 100644 --- a/.gitignore +++ b/.gitignore @@ -59,4 +59,7 @@ docs/_build/ target/ # Pycharm -.idea/ \ No newline at end of file +.idea/ + +# MacOS +*.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index 3d2df31..524d007 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,15 @@ # datasketch-io + +## Setting your Docker secrets + +The docker compose file for the Datasketch stack requires that the `pg_passwd_datasketch` and `datasketch_key` secret be set before deployment. + +You will need to ssh into the node manager and create these secrets: +``` +echo "mysecret" | docker secret create - +``` + ## Build docker image `docker build -t androiddrew/datasketch:latest -f ./services/cms/Dockerfile ./services/cms` @@ -32,4 +42,12 @@ local replication all trust host replication all 127.0.0.1/32 trust host replication all ::1/128 trust host datasketch datasketch host.docker.internal trust +``` + +## Trouble shooting + +You can run the following command to get the logs for a task, running or not. + +``` +docker logs $(docker inspect --format "{{.Status.ContainerStatus.ContainerID}}" ) ``` \ No newline at end of file diff --git a/config/nginx/conf.d/datasketch.http.conf b/config/nginx/conf.d/datasketch.http.conf new file mode 100644 index 0000000..83b888d --- /dev/null +++ b/config/nginx/conf.d/datasketch.http.conf @@ -0,0 +1,7 @@ +# Datasketch.io + +upstream datasketch_app { + server swarm1.androiddrew.com:8000; + server swarm2.androiddrew.com:8000; + server swarm3.androiddrew.com:8000; +} \ No newline at end of file diff --git a/config/nginx/sites-available/datasketch.io b/config/nginx/sites-available/datasketch.io new file mode 100644 index 0000000..5138e7a --- /dev/null +++ b/config/nginx/sites-available/datasketch.io @@ -0,0 +1,40 @@ +# Datasketch.io +# Upstream set datasketch_app in /etc/nginx/conf.d/datasketch.http.conf + +server { + # HTTP CONFIG + server_name datasketch.io; + + + root /var/www/datasketch; + # index index.html; + + location / { + + proxy_pass http://datasketch_app; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + # try_files $uri $uri/ =404; + + } + + listen 443 ssl; # managed by Certbot + ssl_certificate /etc/letsencrypt/live/datasketch.io/fullchain.pem; # managed by Certbot + ssl_certificate_key /etc/letsencrypt/live/datasketch.io/privkey.pem; # managed by Certbot + include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot + +} + +server { + if ($host = datasketch.io) { + return 301 https://$host$request_uri; + } # managed by Certbot + + + listen 80; + server_name datasketch.io; + return 404; # managed by Certbot + + +} \ No newline at end of file diff --git a/docker-compose-swarm.yml b/docker-compose-swarm.yml new file mode 100644 index 0000000..03da986 --- /dev/null +++ b/docker-compose-swarm.yml @@ -0,0 +1,70 @@ +version: '3.6' + +services: + + cms: + image: androiddrew/datasketch:latest + deploy: + replicas: 1 + restart_policy: + condition: on-failure + ports: + - 8000:8000 + secrets: + - datasketch_key + - pg_passwd_datasketch + environment: + - DJANGO_SETTINGS_MODULE=cms.settings.production + - DB_HOST=db + - DB_USER=datasketch + volumes: + - datasketch-media-vol:/code/media + depends_on: + - db + networks: + - datasketch_net + + db: + image: postgres:10.5-alpine + deploy: + replicas: 1 + restart_policy: + condition: on-failure + # Docker secrets for postgres are referenced by /run/secrets/ + secrets: + - pg_passwd_datasketch + environment: + - POSTGRES_USER=datasketch + - POSTGRES_PASSWORD_FILE=/run/secrets/pg_passwd_datasketch + - POSTGRES_DB=datasketch + expose: + - 5432 + volumes: + - datasketch-db-vol:/var/lib/postgresql/data + networks: + - datasketch_net + + +networks: + datasketch_net: + driver: overlay + +volumes: + datasketch-db-vol: + driver: local + driver_opts: + type: nfs + o: addr=nas1.androiddrew.com,rw + device: ":/volume1/expanse/datasketch_pg_data" + datasketch-media-vol: + driver: local + driver_opts: + type: nfs + o: addr=nas1.androiddrew.com,rw + device: ":/volume1/expanse/datasketch_media" + +secrets: + pg_passwd_datasketch: + external: true + datasketch_key: + external: true \ No newline at end of file diff --git a/services/cms/Dockerfile b/services/cms/Dockerfile index d99ba7a..79d9701 100644 --- a/services/cms/Dockerfile +++ b/services/cms/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.7 LABEL maintainer="drew@androiddrew.comw" ENV PYTHONUNBUFFERED 1 -ENV DJANGO_ENV dev +ENV DJANGO_ENV production COPY ./requirements.txt /code/requirements.txt RUN pip install -r /code/requirements.txt @@ -11,7 +11,7 @@ RUN pip install gunicorn COPY . /code/ WORKDIR /code/ -# RUN python manage.py migrate # removed to garuantee build +RUN python manage.py collectstatic RUN useradd wagtail RUN chown -R wagtail /code diff --git a/services/cms/README.md b/services/cms/README.md new file mode 100644 index 0000000..11ca287 --- /dev/null +++ b/services/cms/README.md @@ -0,0 +1,26 @@ +# Datasketch CMS + +This service is built using the [Wagtail](http://docs.wagtail.io/en/v2.3/) framework. + +## Middleware + +Since this application is being deployed to a Docker swarm it utilizes [Whitenoise](http://whitenoise.evans.io/en/stable/) and the Django specific whitenoise middleware to deliver static content. This solution is suitable for serving low traffic static content. To improve performance an Nginx reverse proxy could leverage HTTP caching or could be configured to serve all static content directly. + +## Environmental Variables + +The follow environment variables must be set for development. Note that the DB_PASSWD for production must be a docker secret. + +``` +DB_HOST= +DB_PASSWD=ec +``` + +In production you must set the `DJANGO_SETTINGS_MODULE` to the appropriate module: + +``` +DJANGO_SETTINGS_MODULE=cms.settings.production +``` + +## Migrations + +The `manage.py` file is used to upgrade the backend database. You will need to log into the container on the appropriate host and execute the command `python manage.py migrate` manually. diff --git a/services/cms/cms/settings/base.py b/services/cms/cms/settings/base.py index 2064cd6..a5d3f98 100644 --- a/services/cms/cms/settings/base.py +++ b/services/cms/cms/settings/base.py @@ -48,6 +48,7 @@ INSTALLED_APPS = [ ] MIDDLEWARE = [ + "whitenoise.middleware.WhiteNoiseMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", diff --git a/services/cms/cms/settings/production.py b/services/cms/cms/settings/production.py index 9ca4ed7..e1e9797 100644 --- a/services/cms/cms/settings/production.py +++ b/services/cms/cms/settings/production.py @@ -2,6 +2,42 @@ from .base import * DEBUG = False +# Configured with docker secrets +SECRET_KEY = open("/run/secrets/datasketch_key", "r").read().strip() +PG_PASSWD = open("/run/secrets/pg_passwd_datasketch", "r").read().strip() + +# SECURITY WARNING: define the correct hosts in production! +ALLOWED_HOSTS = [".datasketch.io", ".androiddrew.com", "localhost", "127.0.0.1"] + +EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql", + "NAME": "datasketch", + "USER": "datasketch", + "PASSWORD": PG_PASSWD, + "HOST": os.environ.get("DB_HOST") if os.environ.get("DB_HOST") else "127.0.0.1", + "PORT": "5432", + } +} + +# Set as the volume map for media on the nas +MEDIA_URL = "/code/media/" + +LOGGING = { + "version": 1, + "disable_exisiing_loggers": False, + "handlers": {"console": {"class": "logging.StreamHandler"}}, + "loggers": { + "django.request": { + "handlers": ["console"], + "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"), + } + }, +} + + try: from .local import * except ImportError: diff --git a/services/cms/requirements.txt b/services/cms/requirements.txt index 480ac28..80fe774 100644 --- a/services/cms/requirements.txt +++ b/services/cms/requirements.txt @@ -1,3 +1,4 @@ Django>=2.1,<2.2 wagtail>=2.3,<2.4 -psycopg2>=2.7.5 \ No newline at end of file +psycopg2>=2.7.5 +whitenoise>=4.1 \ No newline at end of file