From 94b3a34a3cf834dceb227c3a83335e62ab8e2b59 Mon Sep 17 00:00:00 2001 From: Drew Bednar Date: Fri, 27 Nov 2020 22:46:23 -0500 Subject: [PATCH] Added some supervisord experiments --- .gitignore | 5 + data_structures/__init__.py | 0 data_structures/buffers.py | 34 +++++ data_structures/utils.py | 32 +++++ supervisord/bin/simple1.py | 18 +++ supervisord/bin/simple2.py | 18 +++ supervisord/etc/supervisord.conf | 36 +++++ supervisord/etc/supervisord.conf.example | 170 +++++++++++++++++++++++ supervisord/requirements.txt | 3 + 9 files changed, 316 insertions(+) create mode 100644 data_structures/__init__.py create mode 100644 data_structures/buffers.py create mode 100644 data_structures/utils.py create mode 100644 supervisord/bin/simple1.py create mode 100644 supervisord/bin/simple2.py create mode 100644 supervisord/etc/supervisord.conf create mode 100644 supervisord/etc/supervisord.conf.example create mode 100644 supervisord/requirements.txt diff --git a/.gitignore b/.gitignore index e15eec3..b95dfec 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,8 @@ target/ #Pycharm .idea/ +# Supervisor +supervisord/supervisord.log +supervisord/supervisord.pid +supervisord/var/supervisor.sock +supervisord/log/* \ No newline at end of file diff --git a/data_structures/__init__.py b/data_structures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/data_structures/buffers.py b/data_structures/buffers.py new file mode 100644 index 0000000..3893c61 --- /dev/null +++ b/data_structures/buffers.py @@ -0,0 +1,34 @@ +import logging + +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger(__name__) + + +class RingBuffer: + """Implements a basic Ring Buffer type.""" + + def __init__(self, size): + self._data = bytearray(size) + self.size = size + self.put_index = 0 + self.get_index = 0 + + def put(self, value): + _next_index = (self.put_index + 1) % self.size + # check for overflow + if self.get_index != _next_index: + self._data[self.put_index] = value + self.put_index = _next_index + logger.debug(f'get_index:{self.get_index}, put_index:{self.put_index}') + logger.debug(str(self._data)) + return value + else: + return None + + def get(self): + if self.get_index == self.put_index: + return None ## buffer empty + else: + value = self._data[self.get_index] + self.get_index = (self.get_index + 1) % self.size + return value diff --git a/data_structures/utils.py b/data_structures/utils.py new file mode 100644 index 0000000..7600e5c --- /dev/null +++ b/data_structures/utils.py @@ -0,0 +1,32 @@ +import sys +from numbers import Number +from collections import Set, Mapping, deque + +zero_depth_bases = (str, bytes, Number, range, bytearray) +iteritems = 'items' + + +def getsize(obj_0): + """Recursively iterate to sum size of object & members.""" + _seen_ids = set() + + def inner(obj): + obj_id = id(obj) + if obj_id in _seen_ids: + return 0 + _seen_ids.add(obj_id) + size = sys.getsizeof(obj) + if isinstance(obj, zero_depth_bases): + pass # bypass remaining control flow and return + elif isinstance(obj, (tuple, list, Set, deque)): + size += sum(inner(i) for i in obj) + elif isinstance(obj, Mapping) or hasattr(obj, iteritems): + size += sum(inner(k) + inner(v) for k, v in getattr(obj, iteritems)()) + # Check for custom object instances - may subclass above too + if hasattr(obj, '__dict__'): + size += inner(vars(obj)) + if hasattr(obj, '__slots__'): # can have __slots__ with __dict__ + size += sum(inner(getattr(obj, s)) for s in obj.__slots__ if hasattr(obj, s)) + return size + + return inner(obj_0) diff --git a/supervisord/bin/simple1.py b/supervisord/bin/simple1.py new file mode 100644 index 0000000..b50d6c1 --- /dev/null +++ b/supervisord/bin/simple1.py @@ -0,0 +1,18 @@ +import logging + +from flask import Flask, request + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +app = Flask(__name__) + + +@app.route('/') +def index(): + log.warning('Got a request from: {}'.format(request.remote_addr)) + return {'msg': 'pong simple1'} + + +if __name__ == '__main__': + app.run(host='0.0.0.0') diff --git a/supervisord/bin/simple2.py b/supervisord/bin/simple2.py new file mode 100644 index 0000000..342f39e --- /dev/null +++ b/supervisord/bin/simple2.py @@ -0,0 +1,18 @@ +import logging + +from flask import Flask, request + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +app = Flask(__name__) + + +@app.route('/') +def index(): + log.info('Got a request from: {}'.format(request.remote_addr)) + return {'msg': 'pong simple2'} + + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=5001) diff --git a/supervisord/etc/supervisord.conf b/supervisord/etc/supervisord.conf new file mode 100644 index 0000000..3a30a25 --- /dev/null +++ b/supervisord/etc/supervisord.conf @@ -0,0 +1,36 @@ +[supervisord] + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[inet_http_server] +port = 127.0.0.1:9001 +username = user +password = 123 + +[unix_http_server] +file=%(here)s/../var/supervisor.sock ; the path to the socket file +;chmod=0700 ; socket file mode (default 0700) +;chown=nobody:nogroup ; socket file uid:gid owner +;username=user ; default is no username (open server) +;password=123 ; default is no password (open server) + +[supervisorctl] +serverurl=unix://%(here)s/../var/supervisor.sock ; use a unix:// URL for a unix socket + +[program:simple1] +;command=python %(here)s/../bin/simple1.py +command=gunicorn -w 4 -b 0.0.0.0:5000 bin.simple1:app +directory=%(here)s/.. +autorestart=yes +startretries=3 +stdout_logfile=%(here)s/../log/simple1.log +redirect_stderr=true + +[program:simple2] +command=gunicorn -w 4 -b 0.0.0.0:5001 bin.simple2:app +directory=%(here)s/.. +autorestart=yes +startretries=3 +stdout_logfile=%(here)s/../log/simple2.log +redirect_stderr=true \ No newline at end of file diff --git a/supervisord/etc/supervisord.conf.example b/supervisord/etc/supervisord.conf.example new file mode 100644 index 0000000..289c9cb --- /dev/null +++ b/supervisord/etc/supervisord.conf.example @@ -0,0 +1,170 @@ +; Sample supervisor config file. +; +; For more information on the config file, please see: +; http://supervisord.org/configuration.html +; +; Notes: +; - Shell expansion ("~" or "$HOME") is not supported. Environment +; variables can be expanded using this syntax: "%(ENV_HOME)s". +; - Quotes around values are not supported, except in the case of +; the environment= options as shown below. +; - Comments must have a leading space: "a=b ;comment" not "a=b;comment". +; - Command will be truncated if it looks like a config file comment, e.g. +; "command=bash -c 'foo ; bar'" will truncate to "command=bash -c 'foo ". +; +; Warning: +; Paths throughout this example file use /tmp because it is available on most +; systems. You will likely need to change these to locations more appropriate +; for your system. Some systems periodically delete older files in /tmp. +; Notably, if the socket file defined in the [unix_http_server] section below +; is deleted, supervisorctl will be unable to connect to supervisord. + +[unix_http_server] +file=/tmp/supervisor.sock ; the path to the socket file +;chmod=0700 ; socket file mode (default 0700) +;chown=nobody:nogroup ; socket file uid:gid owner +;username=user ; default is no username (open server) +;password=123 ; default is no password (open server) + +; Security Warning: +; The inet HTTP server is not enabled by default. The inet HTTP server is +; enabled by uncommenting the [inet_http_server] section below. The inet +; HTTP server is intended for use within a trusted environment only. It +; should only be bound to localhost or only accessible from within an +; isolated, trusted network. The inet HTTP server does not support any +; form of encryption. The inet HTTP server does not use authentication +; by default (see the username= and password= options to add authentication). +; Never expose the inet HTTP server to the public internet. + +;[inet_http_server] ; inet (TCP) server disabled by default +;port=127.0.0.1:9001 ; ip_address:port specifier, *:port for all iface +;username=user ; default is no username (open server) +;password=123 ; default is no password (open server) + +[supervisord] +logfile=/tmp/supervisord.log ; main log file; default $CWD/supervisord.log +logfile_maxbytes=50MB ; max main logfile bytes b4 rotation; default 50MB +logfile_backups=10 ; # of main logfile backups; 0 means none, default 10 +loglevel=info ; log level; default info; others: debug,warn,trace +pidfile=/tmp/supervisord.pid ; supervisord pidfile; default supervisord.pid +nodaemon=false ; start in foreground if true; default false +silent=false ; no logs to stdout if true; default false +minfds=1024 ; min. avail startup file descriptors; default 1024 +minprocs=200 ; min. avail process descriptors;default 200 +;umask=022 ; process file creation umask; default 022 +;user=supervisord ; setuid to this UNIX account at startup; recommended if root +;identifier=supervisor ; supervisord identifier, default is 'supervisor' +;directory=/tmp ; default is not to cd during start +;nocleanup=true ; don't clean up tempfiles at start; default false +;childlogdir=/tmp ; 'AUTO' child log dir, default $TEMP +;environment=KEY="value" ; key value pairs to add to environment +;strip_ansi=false ; strip ansi escape codes in logs; def. false + +; The rpcinterface:supervisor section must remain in the config file for +; RPC (supervisorctl/web interface) to work. Additional interfaces may be +; added by defining them in separate [rpcinterface:x] sections. + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +; The supervisorctl section configures how supervisorctl will connect to +; supervisord. configure it match the settings in either the unix_http_server +; or inet_http_server section. + +[supervisorctl] +serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket +;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket +;username=chris ; should be same as in [*_http_server] if set +;password=123 ; should be same as in [*_http_server] if set +;prompt=mysupervisor ; cmd line prompt (default "supervisor") +;history_file=~/.sc_history ; use readline history if available + +; The sample program section below shows all possible program subsection values. +; Create one or more 'real' program: sections to be able to control them under +; supervisor. + +;[program:theprogramname] +;command=/bin/cat ; the program (relative uses PATH, can take args) +;process_name=%(program_name)s ; process_name expr (default %(program_name)s) +;numprocs=1 ; number of processes copies to start (def 1) +;directory=/tmp ; directory to cwd to before exec (def no cwd) +;umask=022 ; umask for process (default None) +;priority=999 ; the relative start priority (default 999) +;autostart=true ; start at supervisord start (default: true) +;startsecs=1 ; # of secs prog must stay up to be running (def. 1) +;startretries=3 ; max # of serial start failures when starting (default 3) +;autorestart=unexpected ; when to restart if exited after running (def: unexpected) +;exitcodes=0 ; 'expected' exit codes used with autorestart (default 0) +;stopsignal=QUIT ; signal used to kill process (default TERM) +;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) +;stopasgroup=false ; send stop signal to the UNIX process group (default false) +;killasgroup=false ; SIGKILL the UNIX process group (def false) +;user=chrism ; setuid to this UNIX account to run the program +;redirect_stderr=true ; redirect proc stderr to stdout (default false) +;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO +;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +;stdout_logfile_backups=10 ; # of stdout logfile backups (0 means none, default 10) +;stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) +;stdout_events_enabled=false ; emit events on stdout writes (default false) +;stdout_syslog=false ; send stdout to syslog with process name (default false) +;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO +;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +;stderr_logfile_backups=10 ; # of stderr logfile backups (0 means none, default 10) +;stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) +;stderr_events_enabled=false ; emit events on stderr writes (default false) +;stderr_syslog=false ; send stderr to syslog with process name (default false) +;environment=A="1",B="2" ; process environment additions (def no adds) +;serverurl=AUTO ; override serverurl computation (childutils) + +; The sample eventlistener section below shows all possible eventlistener +; subsection values. Create one or more 'real' eventlistener: sections to be +; able to handle event notifications sent by supervisord. + +;[eventlistener:theeventlistenername] +;command=/bin/eventlistener ; the program (relative uses PATH, can take args) +;process_name=%(program_name)s ; process_name expr (default %(program_name)s) +;numprocs=1 ; number of processes copies to start (def 1) +;events=EVENT ; event notif. types to subscribe to (req'd) +;buffer_size=10 ; event buffer queue size (default 10) +;directory=/tmp ; directory to cwd to before exec (def no cwd) +;umask=022 ; umask for process (default None) +;priority=-1 ; the relative start priority (default -1) +;autostart=true ; start at supervisord start (default: true) +;startsecs=1 ; # of secs prog must stay up to be running (def. 1) +;startretries=3 ; max # of serial start failures when starting (default 3) +;autorestart=unexpected ; autorestart if exited after running (def: unexpected) +;exitcodes=0 ; 'expected' exit codes used with autorestart (default 0) +;stopsignal=QUIT ; signal used to kill process (default TERM) +;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) +;stopasgroup=false ; send stop signal to the UNIX process group (default false) +;killasgroup=false ; SIGKILL the UNIX process group (def false) +;user=chrism ; setuid to this UNIX account to run the program +;redirect_stderr=false ; redirect_stderr=true is not allowed for eventlisteners +;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO +;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +;stdout_logfile_backups=10 ; # of stdout logfile backups (0 means none, default 10) +;stdout_events_enabled=false ; emit events on stdout writes (default false) +;stdout_syslog=false ; send stdout to syslog with process name (default false) +;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO +;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB) +;stderr_logfile_backups=10 ; # of stderr logfile backups (0 means none, default 10) +;stderr_events_enabled=false ; emit events on stderr writes (default false) +;stderr_syslog=false ; send stderr to syslog with process name (default false) +;environment=A="1",B="2" ; process environment additions +;serverurl=AUTO ; override serverurl computation (childutils) + +; The sample group section below shows all possible group values. Create one +; or more 'real' group: sections to create "heterogeneous" process groups. + +;[group:thegroupname] +;programs=progname1,progname2 ; each refers to 'x' in [program:x] definitions +;priority=999 ; the relative start priority (default 999) + +; The [include] section can just contain the "files" setting. This +; setting can list multiple files (separated by whitespace or +; newlines). It can also contain wildcards. The filenames are +; interpreted as relative to this file. Included files *cannot* +; include files themselves. + +;[include] +;files = relative/directory/*.ini diff --git a/supervisord/requirements.txt b/supervisord/requirements.txt new file mode 100644 index 0000000..f941d23 --- /dev/null +++ b/supervisord/requirements.txt @@ -0,0 +1,3 @@ +flask +gunicorn +requests