From 454e1714c81fd31b76771fe2678f6f72e9fcb1e7 Mon Sep 17 00:00:00 2001 From: Eyob Yirdaw Date: Fri, 15 Feb 2019 19:35:39 +0300 Subject: [PATCH 01/10] prepared deployment configs, not finished yet --- Docker/Dockerfile | 4 ++++ Docker/requirements_2.txt | 1 + docs/gunicorn/config.py | 10 ++++++++++ docs/supervisor/supervisord.conf | 30 ++++++++++++++++++++++++++++++ docs/supervisor/topic_grpc.conf | 12 ++++++++++++ docs/supervisor/topic_rest.conf | 12 ++++++++++++ docs/{ => tests}/test_doc.txt | 0 test_topic_analysis_grpc.py | 2 +- 8 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 Docker/requirements_2.txt create mode 100644 docs/gunicorn/config.py create mode 100644 docs/supervisor/supervisord.conf create mode 100644 docs/supervisor/topic_grpc.conf create mode 100644 docs/supervisor/topic_rest.conf rename docs/{ => tests}/test_doc.txt (100%) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index ff5e249..20c84a1 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -36,6 +36,10 @@ RUN sudo pip install -r requirements.txt RUN echo 1 RUN python3.6 -m nltk.downloader all +RUN sudo apt-get install -y supervisor + +COPY requirements_2.txt /tmp/ +RUN sudo pip install -r requirements_2.txt WORKDIR /home/top/ RUN mkdir dep diff --git a/Docker/requirements_2.txt b/Docker/requirements_2.txt new file mode 100644 index 0000000..8f22dcc --- /dev/null +++ b/Docker/requirements_2.txt @@ -0,0 +1 @@ +gunicorn diff --git a/docs/gunicorn/config.py b/docs/gunicorn/config.py new file mode 100644 index 0000000..381e243 --- /dev/null +++ b/docs/gunicorn/config.py @@ -0,0 +1,10 @@ +workers = 3 +bind = "0.0.0.0:4998" +errorlog = "/home/top/gunicorn/error.log" +accesslog = "/home/top/gunicorn/access.log" +loglevel = "debug" +max_requests = 10 +pidfile = "/home/top/gunicorn/pid.pid" +chdir = "/home/top/dep/topic-analysis" +timeout = 120 +access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" "%(m)s" "%(U)s" "%(q)s" "%(H)s" %(B)s %(T)s %(p)s "%({Header}i)s" "%({Header}o)s" "%({Header}e)s"' diff --git a/docs/supervisor/supervisord.conf b/docs/supervisor/supervisord.conf new file mode 100644 index 0000000..6e0d485 --- /dev/null +++ b/docs/supervisor/supervisord.conf @@ -0,0 +1,30 @@ +; supervisor config file + +[unix_http_server] +file=/var/run/supervisor.sock ; (the path to the socket file) +chmod=0700 ; sockef file mode (default 0700) + +[supervisord] +logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) +pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) +logfile_maxbytes=10MB +logfile_backups=10 + +; the below section must remain in the config file for RPC +; (supervisorctl/web interface) to work, additional interfaces may be +; added by defining them in separate rpcinterface: sections +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL for a unix socket + +; The [include] section can just contain the "files" setting. This +; setting can list multiple files (separated by whitespace or +; newlines). It can also contain wildcards. The filenames are +; interpreted as relative to this file. Included files *cannot* +; include files themselves. + +[include] +files = /etc/supervisor/conf.d/*.conf diff --git a/docs/supervisor/topic_grpc.conf b/docs/supervisor/topic_grpc.conf new file mode 100644 index 0000000..c9baa3e --- /dev/null +++ b/docs/supervisor/topic_grpc.conf @@ -0,0 +1,12 @@ +[program:topic_grpc] +command=python3.6 /home/top/dep/topic-analysis/topic_analysis_grpc.py +user=top +autostart=true +autorestart=true +stderr_logfile=/var/log/topic_grpc/err.log +stdout_logfile=/var/log/topic_grpc/out.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=100 +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=100 diff --git a/docs/supervisor/topic_rest.conf b/docs/supervisor/topic_rest.conf new file mode 100644 index 0000000..61aa3da --- /dev/null +++ b/docs/supervisor/topic_rest.conf @@ -0,0 +1,12 @@ +[program:topic_rest] +command=gunicorn -c /home/top/dep/topic-analysis/config.py analysis_results:app +user=top +autostart=true +autorestart=true +stderr_logfile=/var/log/topic_rest/err.log +stdout_logfile=/var/log/topic_rest/out.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=100 +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=100 diff --git a/docs/test_doc.txt b/docs/tests/test_doc.txt similarity index 100% rename from docs/test_doc.txt rename to docs/tests/test_doc.txt diff --git a/test_topic_analysis_grpc.py b/test_topic_analysis_grpc.py index 87d569d..46c3913 100644 --- a/test_topic_analysis_grpc.py +++ b/test_topic_analysis_grpc.py @@ -23,7 +23,7 @@ def setUp(self): self.app = analysis_results.app.test_client() self.docs = [] - sample_doc = 'docs/test_doc.txt' + sample_doc = 'docs/tests/test_doc.txt' with open(sample_doc,'r') as f: self.docs = f.read().splitlines() From 8d577d7ef9b407aae95e13d70c870336addb3e6c Mon Sep 17 00:00:00 2001 From: Eyob Yirdaw Date: Mon, 18 Feb 2019 19:29:56 +0300 Subject: [PATCH 02/10] init configs docker file --- .gitignore | 1 + Docker/Dockerfile | 34 +++++++++++++++++++- {docs => Docker}/gunicorn/config.py | 0 Docker/supervisor/kovand.conf | 12 +++++++ Docker/supervisor/ropstend.conf | 12 +++++++ {docs => Docker}/supervisor/supervisord.conf | 0 {docs => Docker}/supervisor/topic_grpc.conf | 0 {docs => Docker}/supervisor/topic_rest.conf | 0 8 files changed, 58 insertions(+), 1 deletion(-) rename {docs => Docker}/gunicorn/config.py (100%) create mode 100644 Docker/supervisor/kovand.conf create mode 100644 Docker/supervisor/ropstend.conf rename {docs => Docker}/supervisor/supervisord.conf (100%) rename {docs => Docker}/supervisor/topic_grpc.conf (100%) rename {docs => Docker}/supervisor/topic_rest.conf (100%) diff --git a/.gitignore b/.gitignore index 115deb1..d96a1d8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ plsa-service/plsa/_plsa.c service_spec/__pycache__/ service_spec/topic_analysis_pb2.py service_spec/topic_analysis_pb2_grpc.py +Docker/daemons/*.* diff --git a/Docker/Dockerfile b/Docker/Dockerfile index 20c84a1..ad489b9 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -36,11 +36,43 @@ RUN sudo pip install -r requirements.txt RUN echo 1 RUN python3.6 -m nltk.downloader all -RUN sudo apt-get install -y supervisor +RUN sudo apt-get install -y supervisor && \ + sudo rm /etc/supervisor/supervisord.conf COPY requirements_2.txt /tmp/ RUN sudo pip install -r requirements_2.txt +RUN sudo mkdir /var/log/kovand/ \ + /var/log/ropstend/ \ + /var/log/topic_grpc/ \ + /var/log/topic_rest/ + +RUN mkdir /home/top/pay/ \ + /home/top/daemons/ \ + /home/top/pay/kovan/ \ + /home/top/pay/ropsten/ \ + /home/top/gunicorn/ \ + /home/top/daemons/kovan/ \ + /home/top/daemons/ropsten/ + + +COPY daemons/snetd_topic_kovan.json /home/top/daemons/kovan/ +COPY daemons/snetd_topic_ropsten.json /home/top/daemons/ropsten/ +COPY gunicorn/config.py /home/top/gunicorn/ +COPY supervisor/kovand.conf /etc/supervisor/conf.d/ +COPY supervisor/ropstend.conf /etc/supervisor/conf.d/ +COPY supervisor/topic_grpc.conf /etc/supervisor/conf.d/ +COPY supervisor/topic_rest.conf /etc/supervisor/conf.d/ +COPY supervisor/supervisord.conf /etc/supervisor/ + +ENV DAEMON_VERSION 0.1.6 + +RUN wget -O snet-daemon.tar.gz https://github.com/singnet/snet-daemon/releases/download/v$DAEMON_VERSION/snet-daemon-v$DAEMON_VERSION-linux-amd64.tar.gz && \ + tar --strip-components=1 -xzf snet-daemon.tar.gz && \ + cp snetd /home/top/daemons/kovan/ && \ + cp snetd /home/top/daemons/ropsten/ + + WORKDIR /home/top/ RUN mkdir dep WORKDIR /home/top/dep diff --git a/docs/gunicorn/config.py b/Docker/gunicorn/config.py similarity index 100% rename from docs/gunicorn/config.py rename to Docker/gunicorn/config.py diff --git a/Docker/supervisor/kovand.conf b/Docker/supervisor/kovand.conf new file mode 100644 index 0000000..f96d40e --- /dev/null +++ b/Docker/supervisor/kovand.conf @@ -0,0 +1,12 @@ +[program:kovand] +command=home/top/daemons/kovan/snetd serve --config /home/top/daemons/kovan/snetd_topic_kovan.json +user=top +autostart=true +autorestart=true +stderr_logfile=/var/log/kovand/err.log +stdout_logfile=/var/log/kovand/out.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=100 +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=100 diff --git a/Docker/supervisor/ropstend.conf b/Docker/supervisor/ropstend.conf new file mode 100644 index 0000000..347cdf5 --- /dev/null +++ b/Docker/supervisor/ropstend.conf @@ -0,0 +1,12 @@ +[program:ropstend] +command=home/top/daemons/ropsten/snetd serve --config /home/top/daemons/ropsten/snetd_topic_ropsten.json +user=top +autostart=true +autorestart=true +stderr_logfile=/var/log/ropstend/err.log +stdout_logfile=/var/log/ropstend/out.log +redirect_stderr=true +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=100 +stderr_logfile_maxbytes=10MB +stderr_logfile_backups=100 diff --git a/docs/supervisor/supervisord.conf b/Docker/supervisor/supervisord.conf similarity index 100% rename from docs/supervisor/supervisord.conf rename to Docker/supervisor/supervisord.conf diff --git a/docs/supervisor/topic_grpc.conf b/Docker/supervisor/topic_grpc.conf similarity index 100% rename from docs/supervisor/topic_grpc.conf rename to Docker/supervisor/topic_grpc.conf diff --git a/docs/supervisor/topic_rest.conf b/Docker/supervisor/topic_rest.conf similarity index 100% rename from docs/supervisor/topic_rest.conf rename to Docker/supervisor/topic_rest.conf From d61a7742b89eea170f91102455e0b65e3139470b Mon Sep 17 00:00:00 2001 From: Eyob Yirdaw Date: Tue, 19 Feb 2019 10:23:51 +0300 Subject: [PATCH 03/10] exp with docker and supervisor --- Docker/Dockerfile | 24 ++++++++++++-------- Docker/supervisor/kovand.conf | 20 ++++++++++------- Docker/supervisor/ropstend.conf | 20 ++++++++++------- Docker/supervisor/supervisord.conf | 4 ++-- Docker/supervisor/topic_grpc.conf | 21 ++++++++++++------ Docker/supervisor/topic_rest.conf | 21 ++++++++++++------ analysis_results.py | 2 +- snet_test_client.py | 35 ++++++++++++++++++++++++++++-- 8 files changed, 103 insertions(+), 44 deletions(-) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index ad489b9..b21ba89 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -55,23 +55,24 @@ RUN mkdir /home/top/pay/ \ /home/top/daemons/kovan/ \ /home/top/daemons/ropsten/ +ENV DAEMON_VERSION 0.1.6 + +RUN wget -O snet-daemon.tar.gz https://github.com/singnet/snet-daemon/releases/download/v$DAEMON_VERSION/snet-daemon-v$DAEMON_VERSION-linux-amd64.tar.gz && \ + tar --strip-components=1 -xzf snet-daemon.tar.gz && \ + cp snetd /home/top/daemons/kovan/ && \ + cp snetd /home/top/daemons/ropsten/ + +RUN echo 22 COPY daemons/snetd_topic_kovan.json /home/top/daemons/kovan/ COPY daemons/snetd_topic_ropsten.json /home/top/daemons/ropsten/ COPY gunicorn/config.py /home/top/gunicorn/ -COPY supervisor/kovand.conf /etc/supervisor/conf.d/ -COPY supervisor/ropstend.conf /etc/supervisor/conf.d/ +#COPY supervisor/kovand.conf /etc/supervisor/conf.d/ +#COPY supervisor/ropstend.conf /etc/supervisor/conf.d/ COPY supervisor/topic_grpc.conf /etc/supervisor/conf.d/ COPY supervisor/topic_rest.conf /etc/supervisor/conf.d/ COPY supervisor/supervisord.conf /etc/supervisor/ -ENV DAEMON_VERSION 0.1.6 - -RUN wget -O snet-daemon.tar.gz https://github.com/singnet/snet-daemon/releases/download/v$DAEMON_VERSION/snet-daemon-v$DAEMON_VERSION-linux-amd64.tar.gz && \ - tar --strip-components=1 -xzf snet-daemon.tar.gz && \ - cp snetd /home/top/daemons/kovan/ && \ - cp snetd /home/top/daemons/ropsten/ - WORKDIR /home/top/ RUN mkdir dep @@ -90,4 +91,9 @@ WORKDIR /home/top/dep/topic-analysis RUN pwd RUN python3.6 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. service_spec/topic_analysis.proto +#CMD ["/usr/bin/supervisord"] + +#RUN sudo service supervisor stop +#RUN sudo service supervisor start + diff --git a/Docker/supervisor/kovand.conf b/Docker/supervisor/kovand.conf index f96d40e..2a159e4 100644 --- a/Docker/supervisor/kovand.conf +++ b/Docker/supervisor/kovand.conf @@ -1,12 +1,16 @@ +;[supervisord] +;nodaemon=true + [program:kovand] -command=home/top/daemons/kovan/snetd serve --config /home/top/daemons/kovan/snetd_topic_kovan.json +command=./serve --config /home/top/daemons/kovan/snetd_topic_kovan.json +directory=/home/top/daemons/kovan/snetd user=top autostart=true autorestart=true -stderr_logfile=/var/log/kovand/err.log -stdout_logfile=/var/log/kovand/out.log -redirect_stderr=true -stdout_logfile_maxbytes=10MB -stdout_logfile_backups=100 -stderr_logfile_maxbytes=10MB -stderr_logfile_backups=100 +;stderr_logfile=/var/log/kovand/err.log +;stdout_logfile=/var/log/kovand/out.log +;redirect_stderr=true +;stdout_logfile_maxbytes=10MB +;stdout_logfile_backups=100 +;stderr_logfile_maxbytes=10MB +;stderr_logfile_backups=100 diff --git a/Docker/supervisor/ropstend.conf b/Docker/supervisor/ropstend.conf index 347cdf5..8a9715f 100644 --- a/Docker/supervisor/ropstend.conf +++ b/Docker/supervisor/ropstend.conf @@ -1,12 +1,16 @@ +;[supervisord] +;nodaemon=true + [program:ropstend] -command=home/top/daemons/ropsten/snetd serve --config /home/top/daemons/ropsten/snetd_topic_ropsten.json +command=./serve --config /home/top/daemons/ropsten/snetd_topic_ropsten.json +directory=/home/top/daemons/ropsten/snetd user=top autostart=true autorestart=true -stderr_logfile=/var/log/ropstend/err.log -stdout_logfile=/var/log/ropstend/out.log -redirect_stderr=true -stdout_logfile_maxbytes=10MB -stdout_logfile_backups=100 -stderr_logfile_maxbytes=10MB -stderr_logfile_backups=100 +;stderr_logfile=/var/log/ropstend/err.log +;stdout_logfile=/var/log/ropstend/out.log +;redirect_stderr=true +;stdout_logfile_maxbytes=10MB +;stdout_logfile_backups=100 +;stderr_logfile_maxbytes=10MB +;stderr_logfile_backups=100 diff --git a/Docker/supervisor/supervisord.conf b/Docker/supervisor/supervisord.conf index 6e0d485..0b574b3 100644 --- a/Docker/supervisor/supervisord.conf +++ b/Docker/supervisor/supervisord.conf @@ -8,8 +8,8 @@ chmod=0700 ; sockef file mode (default 0700) logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) -logfile_maxbytes=10MB -logfile_backups=10 +;logfile_maxbytes=10MB +;logfile_backups=10 ; the below section must remain in the config file for RPC ; (supervisorctl/web interface) to work, additional interfaces may be diff --git a/Docker/supervisor/topic_grpc.conf b/Docker/supervisor/topic_grpc.conf index c9baa3e..1343b8f 100644 --- a/Docker/supervisor/topic_grpc.conf +++ b/Docker/supervisor/topic_grpc.conf @@ -1,12 +1,19 @@ +;[supervisord] +;nodaemon=true + [program:topic_grpc] -command=python3.6 /home/top/dep/topic-analysis/topic_analysis_grpc.py +command=python3.6 topic_analysis_grpc.py +directory=/home/top/dep/topic-analysis/ user=top autostart=true autorestart=true -stderr_logfile=/var/log/topic_grpc/err.log -stdout_logfile=/var/log/topic_grpc/out.log +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 redirect_stderr=true -stdout_logfile_maxbytes=10MB -stdout_logfile_backups=100 -stderr_logfile_maxbytes=10MB -stderr_logfile_backups=100 +;stderr_logfile=/var/log/topic_grpc/err.log +;stdout_logfile=/var/log/topic_grpc/out.log +;redirect_stderr=true +;stdout_logfile_maxbytes=10MB +;stdout_logfile_backups=100 +;stderr_logfile_maxbytes=10MB +;stderr_logfile_backups=100 diff --git a/Docker/supervisor/topic_rest.conf b/Docker/supervisor/topic_rest.conf index 61aa3da..211c811 100644 --- a/Docker/supervisor/topic_rest.conf +++ b/Docker/supervisor/topic_rest.conf @@ -1,12 +1,19 @@ +;[supervisord] +;nodaemon=true + [program:topic_rest] -command=gunicorn -c /home/top/dep/topic-analysis/config.py analysis_results:app +command=gunicorn -c config.py analysis_results:app +directory=/home/top/gunicorn/ user=top autostart=true autorestart=true -stderr_logfile=/var/log/topic_rest/err.log -stdout_logfile=/var/log/topic_rest/out.log +stdout_logfile=/dev/fd/2 +stdout_logfile_maxbytes=0 redirect_stderr=true -stdout_logfile_maxbytes=10MB -stdout_logfile_backups=100 -stderr_logfile_maxbytes=10MB -stderr_logfile_backups=100 +;stderr_logfile=/var/log/topic_rest/err.log +;stdout_logfile=/var/log/topic_rest/out.log +;redirect_stderr=true +;stdout_logfile_maxbytes=10MB +;stdout_logfile_backups=100 +;stderr_logfile_maxbytes=10MB +;stderr_logfile_backups=100 diff --git a/analysis_results.py b/analysis_results.py index 149a4e7..d601bc6 100644 --- a/analysis_results.py +++ b/analysis_results.py @@ -137,4 +137,4 @@ def not_found(error): # app.run(debug=True) - app.run(debug=False,port=4999) + app.run(debug=False,port=4998) diff --git a/snet_test_client.py b/snet_test_client.py index 7c4866a..3042b52 100644 --- a/snet_test_client.py +++ b/snet_test_client.py @@ -30,6 +30,20 @@ def sample_data(): return docs +def sample_data_2(): + + path = str(pathlib.Path(os.path.abspath('')).parents[0])+'/topic-analysis/docs/tests/test_doc.txt' + + docs = [] + + with open(path, "r") as read_file: + text = read_file.read().splitlines() + + for k in text: + if k != "": + docs.append(k) + + return docs def csv_reader(): @@ -58,7 +72,7 @@ def csv_reader(): def try_plsa(): - channel = grpc.insecure_channel('localhost:5000') + channel = grpc.insecure_channel('localhost:5001') # channel = grpc.insecure_channel('172.17.0.75:5001') stub = topic_analysis_pb2_grpc.TopicAnalysisStub(channel) @@ -75,6 +89,22 @@ def try_plsa(): print(resp.handle) +def try_plsa_2(): + channel = grpc.insecure_channel('localhost:5000') + # channel = grpc.insecure_channel('172.17.0.75:5001') + stub = topic_analysis_pb2_grpc.TopicAnalysisStub(channel) + + + plsa_request = topic_analysis_pb2.PLSARequest(docs=sample_data_2(),num_topics=2,maxiter=22,beta=1) + + resp = stub.PLSA(plsa_request) + + + print(resp.status) + print('^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^') + print(resp.message) + print('^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^') + print(resp.handle) @@ -83,7 +113,8 @@ def try_plsa(): if __name__ == '__main__': - try_plsa() + # try_plsa() + try_plsa_2() # csv_reader() From 59f141c6d9e03ce5d2d2df608f0cc56d6882dadd Mon Sep 17 00:00:00 2001 From: Eyob Yirdaw Date: Tue, 19 Feb 2019 17:11:15 +0300 Subject: [PATCH 04/10] Finished dockerfile conf, logging from supervisor to be polished further at a later date --- Docker/Dockerfile | 23 ++++++++++++++++++----- Docker/supervisor/kovand.conf | 8 ++++++-- Docker/supervisor/ropstend.conf | 8 ++++++-- Docker/supervisor/topic_grpc.conf | 7 ++++--- Docker/supervisor/topic_rest.conf | 7 ++++--- snet_test_client.py | 3 ++- 6 files changed, 40 insertions(+), 16 deletions(-) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index b21ba89..229f1e8 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -53,7 +53,15 @@ RUN mkdir /home/top/pay/ \ /home/top/pay/ropsten/ \ /home/top/gunicorn/ \ /home/top/daemons/kovan/ \ - /home/top/daemons/ropsten/ + /home/top/daemons/ropsten/ \ + /home/top/dep/ \ + /home/top/dep/appData/ \ + /home/top/dep/appData/plsa/ \ + /home/top/dep/appData/plsa/cleaned/ \ + /home/top/dep/appData/plsa/dict/ \ + /home/top/dep/appData/plsa/extracted/ \ + /home/top/dep/appData/plsa/plsa-parameters + ENV DAEMON_VERSION 0.1.6 @@ -62,7 +70,15 @@ RUN wget -O snet-daemon.tar.gz https://github.com/singnet/snet-daemon/releases/d cp snetd /home/top/daemons/kovan/ && \ cp snetd /home/top/daemons/ropsten/ -RUN echo 22 +RUN sudo apt-get update && DEBIAN_FRONTEND=noninteractive sudo apt-get install -y locales + +RUN sudo sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + sudo dpkg-reconfigure --frontend=noninteractive locales && \ + sudo update-locale LANG=en_US.UTF-8 + +ENV LANG en_US.UTF-8 + +RUN echo 32 COPY daemons/snetd_topic_kovan.json /home/top/daemons/kovan/ COPY daemons/snetd_topic_ropsten.json /home/top/daemons/ropsten/ @@ -74,8 +90,6 @@ COPY supervisor/topic_rest.conf /etc/supervisor/conf.d/ COPY supervisor/supervisord.conf /etc/supervisor/ -WORKDIR /home/top/ -RUN mkdir dep WORKDIR /home/top/dep #RUN git clone https://github.com/singnet/network-analytics-services.git && \ @@ -96,4 +110,3 @@ RUN python3.6 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. servi #RUN sudo service supervisor stop #RUN sudo service supervisor start - diff --git a/Docker/supervisor/kovand.conf b/Docker/supervisor/kovand.conf index 2a159e4..598e74a 100644 --- a/Docker/supervisor/kovand.conf +++ b/Docker/supervisor/kovand.conf @@ -1,5 +1,6 @@ -;[supervisord] -;nodaemon=true +[supervisord] +logfile=/dev/null +nodaemon=true [program:kovand] command=./serve --config /home/top/daemons/kovan/snetd_topic_kovan.json @@ -7,6 +8,9 @@ directory=/home/top/daemons/kovan/snetd user=top autostart=true autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +redirect_stderr=true ;stderr_logfile=/var/log/kovand/err.log ;stdout_logfile=/var/log/kovand/out.log ;redirect_stderr=true diff --git a/Docker/supervisor/ropstend.conf b/Docker/supervisor/ropstend.conf index 8a9715f..7725ad9 100644 --- a/Docker/supervisor/ropstend.conf +++ b/Docker/supervisor/ropstend.conf @@ -1,5 +1,6 @@ -;[supervisord] -;nodaemon=true +[supervisord] +logfile=/dev/null +nodaemon=true [program:ropstend] command=./serve --config /home/top/daemons/ropsten/snetd_topic_ropsten.json @@ -7,6 +8,9 @@ directory=/home/top/daemons/ropsten/snetd user=top autostart=true autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +redirect_stderr=true ;stderr_logfile=/var/log/ropstend/err.log ;stdout_logfile=/var/log/ropstend/out.log ;redirect_stderr=true diff --git a/Docker/supervisor/topic_grpc.conf b/Docker/supervisor/topic_grpc.conf index 1343b8f..69044a0 100644 --- a/Docker/supervisor/topic_grpc.conf +++ b/Docker/supervisor/topic_grpc.conf @@ -1,5 +1,6 @@ -;[supervisord] -;nodaemon=true +[supervisord] +logfile=/dev/null +nodaemon=true [program:topic_grpc] command=python3.6 topic_analysis_grpc.py @@ -7,7 +8,7 @@ directory=/home/top/dep/topic-analysis/ user=top autostart=true autorestart=true -stdout_logfile=/dev/fd/1 +stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 redirect_stderr=true ;stderr_logfile=/var/log/topic_grpc/err.log diff --git a/Docker/supervisor/topic_rest.conf b/Docker/supervisor/topic_rest.conf index 211c811..cf70e88 100644 --- a/Docker/supervisor/topic_rest.conf +++ b/Docker/supervisor/topic_rest.conf @@ -1,5 +1,6 @@ -;[supervisord] -;nodaemon=true +[supervisord] +logfile=/dev/null +nodaemon=true [program:topic_rest] command=gunicorn -c config.py analysis_results:app @@ -7,7 +8,7 @@ directory=/home/top/gunicorn/ user=top autostart=true autorestart=true -stdout_logfile=/dev/fd/2 +stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 redirect_stderr=true ;stderr_logfile=/var/log/topic_rest/err.log diff --git a/snet_test_client.py b/snet_test_client.py index 3042b52..74da4b4 100644 --- a/snet_test_client.py +++ b/snet_test_client.py @@ -90,7 +90,8 @@ def try_plsa(): def try_plsa_2(): - channel = grpc.insecure_channel('localhost:5000') + # channel = grpc.insecure_channel('localhost:5000') + channel = grpc.insecure_channel('172.17.1.53:5001') # channel = grpc.insecure_channel('172.17.0.75:5001') stub = topic_analysis_pb2_grpc.TopicAnalysisStub(channel) From aeef7bedecab5d46313cadefa2b7f2bbd95fa3c6 Mon Sep 17 00:00:00 2001 From: Eyob Yirdaw Date: Tue, 19 Feb 2019 17:16:03 +0300 Subject: [PATCH 05/10] added daemon confs too --- Docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index 229f1e8..a65a1c9 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -83,8 +83,8 @@ RUN echo 32 COPY daemons/snetd_topic_kovan.json /home/top/daemons/kovan/ COPY daemons/snetd_topic_ropsten.json /home/top/daemons/ropsten/ COPY gunicorn/config.py /home/top/gunicorn/ -#COPY supervisor/kovand.conf /etc/supervisor/conf.d/ -#COPY supervisor/ropstend.conf /etc/supervisor/conf.d/ +COPY supervisor/kovand.conf /etc/supervisor/conf.d/ +COPY supervisor/ropstend.conf /etc/supervisor/conf.d/ COPY supervisor/topic_grpc.conf /etc/supervisor/conf.d/ COPY supervisor/topic_rest.conf /etc/supervisor/conf.d/ COPY supervisor/supervisord.conf /etc/supervisor/ From aaa2755986faea14b8ecf3618d39877473770391 Mon Sep 17 00:00:00 2001 From: Eyob Yirdaw Date: Tue, 19 Feb 2019 18:01:07 +0300 Subject: [PATCH 06/10] modified dockerfile for circleci --- Docker/Dockerfile | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/Docker/Dockerfile b/Docker/Dockerfile index a65a1c9..83a1651 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -92,21 +92,28 @@ COPY supervisor/supervisord.conf /etc/supervisor/ WORKDIR /home/top/dep -#RUN git clone https://github.com/singnet/network-analytics-services.git && \ -# cd network-analytics-services -RUN whoami -RUN echo 13 && \ - git clone https://github.com/edyirdaw/topic-analysis.git && cd topic-analysis && git checkout --track origin/dev +# Can be uncommented for local docker building + +#RUN whoami +# +#RUN echo 13 && \ +# git clone https://github.com/edyirdaw/topic-analysis.git && cd topic-analysis && git checkout --track origin/dev +# +#WORKDIR /home/top/dep/topic-analysis +# +#RUN pwd +#RUN python3.6 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. service_spec/topic_analysis.proto + +# End of Can be uncommented for local docker building -WORKDIR /home/top/dep/topic-analysis -RUN pwd -RUN python3.6 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. service_spec/topic_analysis.proto +## Never going to be uncommented #CMD ["/usr/bin/supervisord"] #RUN sudo service supervisor stop #RUN sudo service supervisor start +## End of Never going to be uncommented From da3d3604d2eadda5926e451c5b283a4187e1b720 Mon Sep 17 00:00:00 2001 From: Eyob Yirdaw Date: Tue, 19 Feb 2019 18:11:09 +0300 Subject: [PATCH 07/10] added circleci conf --- .circleci/config.yml | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..4c3f939 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,48 @@ +workflows: + version: 2 + build-deploy: + jobs: + - build: + filters: + branches: + only: + - master + - dev + - deploy: + requires: + - build + filters: + branches: + only: + - master + - dev + + + +version: 2 +jobs: + build: + docker: + - image: edyirdaw/snet_topic_analysis:v2 + working_directory: /home/top/dep + steps: + - checkout + - run: + name: Run unit tests + command: | + echo 'starting ...' + pwd + echo 'unit tests' + python3.6 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. service_spec/topic_analysis.proto + python3.6 test_topic_analysis_grpc.py + + + deploy: + docker: + - image: edyirdaw/snet_topic_analysis:v2 + steps: + - run: + name: Run deploy script + command: | + echo 'starting deploy ...' + ssh -o StrictHostKeyChecking=no $SSH_USER_1@$SSH_HOST_1 "bash /home/eyob/deploy/src_topic.sh" \ No newline at end of file From b947d347b8363d26e44a82a1696462278f5c2bbd Mon Sep 17 00:00:00 2001 From: Eyob Yirdaw Date: Tue, 19 Feb 2019 18:36:30 +0300 Subject: [PATCH 08/10] fix 1 circleci --- .circleci/config.yml | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c3f939..0f5813f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,24 +1,3 @@ -workflows: - version: 2 - build-deploy: - jobs: - - build: - filters: - branches: - only: - - master - - dev - - deploy: - requires: - - build - filters: - branches: - only: - - master - - dev - - - version: 2 jobs: build: @@ -45,4 +24,25 @@ jobs: name: Run deploy script command: | echo 'starting deploy ...' - ssh -o StrictHostKeyChecking=no $SSH_USER_1@$SSH_HOST_1 "bash /home/eyob/deploy/src_topic.sh" \ No newline at end of file + ssh -o StrictHostKeyChecking=no $SSH_USER_1@$SSH_HOST_1 "bash /home/eyob/deploy/src_topic.sh" + + + +workflows: + version: 2 + build-deploy: + jobs: + - build: + filters: + branches: + only: + - master + - dev + - deploy: + requires: + - build + filters: + branches: + only: + - master + - dev \ No newline at end of file From ddbe7c1e39051086f61bb012107edea64af1b01e Mon Sep 17 00:00:00 2001 From: Eyob Yirdaw Date: Tue, 19 Feb 2019 18:40:07 +0300 Subject: [PATCH 09/10] fix 2 circleci --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f5813f..bc80301 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,7 +36,6 @@ workflows: filters: branches: only: - - master - dev - deploy: requires: @@ -44,5 +43,4 @@ workflows: filters: branches: only: - - master - dev \ No newline at end of file From 4526e889724deb5af24ef917ec574540c33e8ae8 Mon Sep 17 00:00:00 2001 From: Eyob Yirdaw Date: Thu, 21 Feb 2019 12:15:59 +0300 Subject: [PATCH 10/10] fix topic_divider logic --- .circleci/config.yml | 2 ++ topic_analysis_grpc.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bc80301..0f5813f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,6 +36,7 @@ workflows: filters: branches: only: + - master - dev - deploy: requires: @@ -43,4 +44,5 @@ workflows: filters: branches: only: + - master - dev \ No newline at end of file diff --git a/topic_analysis_grpc.py b/topic_analysis_grpc.py index e43dd6c..b60c025 100644 --- a/topic_analysis_grpc.py +++ b/topic_analysis_grpc.py @@ -55,7 +55,7 @@ def PLSA(self,request,context): param_error = True message = 'topic_divider parameter can not be a negative nubmer' - if topic_divider != 0 and num_topics < 2: + if topic_divider == 0 and num_topics < 2: param_error = True message = 'Number of topics should be at least two'