diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..0f5813f --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,48 @@ +version: 2 +jobs: + build: + docker: + - image: edyirdaw/snet_topic_analysis:v2 + working_directory: /home/top/dep + steps: + - checkout + - run: + name: Run unit tests + command: | + echo 'starting ...' + pwd + echo 'unit tests' + python3.6 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. service_spec/topic_analysis.proto + python3.6 test_topic_analysis_grpc.py + + + deploy: + docker: + - image: edyirdaw/snet_topic_analysis:v2 + steps: + - run: + name: Run deploy script + command: | + echo 'starting deploy ...' + ssh -o StrictHostKeyChecking=no $SSH_USER_1@$SSH_HOST_1 "bash /home/eyob/deploy/src_topic.sh" + + + +workflows: + version: 2 + build-deploy: + jobs: + - build: + filters: + branches: + only: + - master + - dev + - deploy: + requires: + - build + filters: + branches: + only: + - master + - dev \ No newline at end of file diff --git a/.gitignore b/.gitignore index 115deb1..d96a1d8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ plsa-service/plsa/_plsa.c service_spec/__pycache__/ service_spec/topic_analysis_pb2.py service_spec/topic_analysis_pb2_grpc.py +Docker/daemons/*.* diff --git a/Docker/Dockerfile b/Docker/Dockerfile index ff5e249..83a1651 100644 --- a/Docker/Dockerfile +++ b/Docker/Dockerfile @@ -36,22 +36,84 @@ RUN sudo pip install -r requirements.txt RUN echo 1 RUN python3.6 -m nltk.downloader all +RUN sudo apt-get install -y supervisor && \ + sudo rm /etc/supervisor/supervisord.conf + +COPY requirements_2.txt /tmp/ +RUN sudo pip install -r requirements_2.txt + +RUN sudo mkdir /var/log/kovand/ \ + /var/log/ropstend/ \ + /var/log/topic_grpc/ \ + /var/log/topic_rest/ + +RUN mkdir /home/top/pay/ \ + /home/top/daemons/ \ + /home/top/pay/kovan/ \ + /home/top/pay/ropsten/ \ + /home/top/gunicorn/ \ + /home/top/daemons/kovan/ \ + /home/top/daemons/ropsten/ \ + /home/top/dep/ \ + /home/top/dep/appData/ \ + /home/top/dep/appData/plsa/ \ + /home/top/dep/appData/plsa/cleaned/ \ + /home/top/dep/appData/plsa/dict/ \ + /home/top/dep/appData/plsa/extracted/ \ + /home/top/dep/appData/plsa/plsa-parameters + + +ENV DAEMON_VERSION 0.1.6 + +RUN wget -O snet-daemon.tar.gz https://github.com/singnet/snet-daemon/releases/download/v$DAEMON_VERSION/snet-daemon-v$DAEMON_VERSION-linux-amd64.tar.gz && \ + tar --strip-components=1 -xzf snet-daemon.tar.gz && \ + cp snetd /home/top/daemons/kovan/ && \ + cp snetd /home/top/daemons/ropsten/ + +RUN sudo apt-get update && DEBIAN_FRONTEND=noninteractive sudo apt-get install -y locales + +RUN sudo sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + sudo dpkg-reconfigure --frontend=noninteractive locales && \ + sudo update-locale LANG=en_US.UTF-8 + +ENV LANG en_US.UTF-8 + +RUN echo 32 + +COPY daemons/snetd_topic_kovan.json /home/top/daemons/kovan/ +COPY daemons/snetd_topic_ropsten.json /home/top/daemons/ropsten/ +COPY gunicorn/config.py /home/top/gunicorn/ +COPY supervisor/kovand.conf /etc/supervisor/conf.d/ +COPY supervisor/ropstend.conf /etc/supervisor/conf.d/ +COPY supervisor/topic_grpc.conf /etc/supervisor/conf.d/ +COPY supervisor/topic_rest.conf /etc/supervisor/conf.d/ +COPY supervisor/supervisord.conf /etc/supervisor/ + -WORKDIR /home/top/ -RUN mkdir dep WORKDIR /home/top/dep -#RUN git clone https://github.com/singnet/network-analytics-services.git && \ -# cd network-analytics-services -RUN whoami -RUN echo 13 && \ - git clone https://github.com/edyirdaw/topic-analysis.git && cd topic-analysis && git checkout --track origin/dev +# Can be uncommented for local docker building + +#RUN whoami +# +#RUN echo 13 && \ +# git clone https://github.com/edyirdaw/topic-analysis.git && cd topic-analysis && git checkout --track origin/dev +# +#WORKDIR /home/top/dep/topic-analysis +# +#RUN pwd +#RUN python3.6 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. service_spec/topic_analysis.proto + +# End of Can be uncommented for local docker building + -WORKDIR /home/top/dep/topic-analysis +## Never going to be uncommented -RUN pwd -RUN python3.6 -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. service_spec/topic_analysis.proto +#CMD ["/usr/bin/supervisord"] +#RUN sudo service supervisor stop +#RUN sudo service supervisor start +## End of Never going to be uncommented diff --git a/Docker/gunicorn/config.py b/Docker/gunicorn/config.py new file mode 100644 index 0000000..381e243 --- /dev/null +++ b/Docker/gunicorn/config.py @@ -0,0 +1,10 @@ +workers = 3 +bind = "0.0.0.0:4998" +errorlog = "/home/top/gunicorn/error.log" +accesslog = "/home/top/gunicorn/access.log" +loglevel = "debug" +max_requests = 10 +pidfile = "/home/top/gunicorn/pid.pid" +chdir = "/home/top/dep/topic-analysis" +timeout = 120 +access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" "%(m)s" "%(U)s" "%(q)s" "%(H)s" %(B)s %(T)s %(p)s "%({Header}i)s" "%({Header}o)s" "%({Header}e)s"' diff --git a/Docker/requirements_2.txt b/Docker/requirements_2.txt new file mode 100644 index 0000000..8f22dcc --- /dev/null +++ b/Docker/requirements_2.txt @@ -0,0 +1 @@ +gunicorn diff --git a/Docker/supervisor/kovand.conf b/Docker/supervisor/kovand.conf new file mode 100644 index 0000000..598e74a --- /dev/null +++ b/Docker/supervisor/kovand.conf @@ -0,0 +1,20 @@ +[supervisord] +logfile=/dev/null +nodaemon=true + +[program:kovand] +command=./serve --config /home/top/daemons/kovan/snetd_topic_kovan.json +directory=/home/top/daemons/kovan/snetd +user=top +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +redirect_stderr=true +;stderr_logfile=/var/log/kovand/err.log +;stdout_logfile=/var/log/kovand/out.log +;redirect_stderr=true +;stdout_logfile_maxbytes=10MB +;stdout_logfile_backups=100 +;stderr_logfile_maxbytes=10MB +;stderr_logfile_backups=100 diff --git a/Docker/supervisor/ropstend.conf b/Docker/supervisor/ropstend.conf new file mode 100644 index 0000000..7725ad9 --- /dev/null +++ b/Docker/supervisor/ropstend.conf @@ -0,0 +1,20 @@ +[supervisord] +logfile=/dev/null +nodaemon=true + +[program:ropstend] +command=./serve --config /home/top/daemons/ropsten/snetd_topic_ropsten.json +directory=/home/top/daemons/ropsten/snetd +user=top +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +redirect_stderr=true +;stderr_logfile=/var/log/ropstend/err.log +;stdout_logfile=/var/log/ropstend/out.log +;redirect_stderr=true +;stdout_logfile_maxbytes=10MB +;stdout_logfile_backups=100 +;stderr_logfile_maxbytes=10MB +;stderr_logfile_backups=100 diff --git a/Docker/supervisor/supervisord.conf b/Docker/supervisor/supervisord.conf new file mode 100644 index 0000000..0b574b3 --- /dev/null +++ b/Docker/supervisor/supervisord.conf @@ -0,0 +1,30 @@ +; supervisor config file + +[unix_http_server] +file=/var/run/supervisor.sock ; (the path to the socket file) +chmod=0700 ; sockef file mode (default 0700) + +[supervisord] +logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log) +pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP) +;logfile_maxbytes=10MB +;logfile_backups=10 + +; the below section must remain in the config file for RPC +; (supervisorctl/web interface) to work, additional interfaces may be +; added by defining them in separate rpcinterface: sections +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL for a unix socket + +; The [include] section can just contain the "files" setting. This +; setting can list multiple files (separated by whitespace or +; newlines). It can also contain wildcards. The filenames are +; interpreted as relative to this file. Included files *cannot* +; include files themselves. + +[include] +files = /etc/supervisor/conf.d/*.conf diff --git a/Docker/supervisor/topic_grpc.conf b/Docker/supervisor/topic_grpc.conf new file mode 100644 index 0000000..69044a0 --- /dev/null +++ b/Docker/supervisor/topic_grpc.conf @@ -0,0 +1,20 @@ +[supervisord] +logfile=/dev/null +nodaemon=true + +[program:topic_grpc] +command=python3.6 topic_analysis_grpc.py +directory=/home/top/dep/topic-analysis/ +user=top +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +redirect_stderr=true +;stderr_logfile=/var/log/topic_grpc/err.log +;stdout_logfile=/var/log/topic_grpc/out.log +;redirect_stderr=true +;stdout_logfile_maxbytes=10MB +;stdout_logfile_backups=100 +;stderr_logfile_maxbytes=10MB +;stderr_logfile_backups=100 diff --git a/Docker/supervisor/topic_rest.conf b/Docker/supervisor/topic_rest.conf new file mode 100644 index 0000000..cf70e88 --- /dev/null +++ b/Docker/supervisor/topic_rest.conf @@ -0,0 +1,20 @@ +[supervisord] +logfile=/dev/null +nodaemon=true + +[program:topic_rest] +command=gunicorn -c config.py analysis_results:app +directory=/home/top/gunicorn/ +user=top +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +redirect_stderr=true +;stderr_logfile=/var/log/topic_rest/err.log +;stdout_logfile=/var/log/topic_rest/out.log +;redirect_stderr=true +;stdout_logfile_maxbytes=10MB +;stdout_logfile_backups=100 +;stderr_logfile_maxbytes=10MB +;stderr_logfile_backups=100 diff --git a/analysis_results.py b/analysis_results.py index 149a4e7..d601bc6 100644 --- a/analysis_results.py +++ b/analysis_results.py @@ -137,4 +137,4 @@ def not_found(error): # app.run(debug=True) - app.run(debug=False,port=4999) + app.run(debug=False,port=4998) diff --git a/docs/test_doc.txt b/docs/tests/test_doc.txt similarity index 100% rename from docs/test_doc.txt rename to docs/tests/test_doc.txt diff --git a/snet_test_client.py b/snet_test_client.py index 7c4866a..74da4b4 100644 --- a/snet_test_client.py +++ b/snet_test_client.py @@ -30,6 +30,20 @@ def sample_data(): return docs +def sample_data_2(): + + path = str(pathlib.Path(os.path.abspath('')).parents[0])+'/topic-analysis/docs/tests/test_doc.txt' + + docs = [] + + with open(path, "r") as read_file: + text = read_file.read().splitlines() + + for k in text: + if k != "": + docs.append(k) + + return docs def csv_reader(): @@ -58,7 +72,7 @@ def csv_reader(): def try_plsa(): - channel = grpc.insecure_channel('localhost:5000') + channel = grpc.insecure_channel('localhost:5001') # channel = grpc.insecure_channel('172.17.0.75:5001') stub = topic_analysis_pb2_grpc.TopicAnalysisStub(channel) @@ -75,6 +89,23 @@ def try_plsa(): print(resp.handle) +def try_plsa_2(): + # channel = grpc.insecure_channel('localhost:5000') + channel = grpc.insecure_channel('172.17.1.53:5001') + # channel = grpc.insecure_channel('172.17.0.75:5001') + stub = topic_analysis_pb2_grpc.TopicAnalysisStub(channel) + + + plsa_request = topic_analysis_pb2.PLSARequest(docs=sample_data_2(),num_topics=2,maxiter=22,beta=1) + + resp = stub.PLSA(plsa_request) + + + print(resp.status) + print('^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^') + print(resp.message) + print('^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^') + print(resp.handle) @@ -83,7 +114,8 @@ def try_plsa(): if __name__ == '__main__': - try_plsa() + # try_plsa() + try_plsa_2() # csv_reader() diff --git a/test_topic_analysis_grpc.py b/test_topic_analysis_grpc.py index 87d569d..46c3913 100644 --- a/test_topic_analysis_grpc.py +++ b/test_topic_analysis_grpc.py @@ -23,7 +23,7 @@ def setUp(self): self.app = analysis_results.app.test_client() self.docs = [] - sample_doc = 'docs/test_doc.txt' + sample_doc = 'docs/tests/test_doc.txt' with open(sample_doc,'r') as f: self.docs = f.read().splitlines() diff --git a/topic_analysis_grpc.py b/topic_analysis_grpc.py index e43dd6c..b60c025 100644 --- a/topic_analysis_grpc.py +++ b/topic_analysis_grpc.py @@ -55,7 +55,7 @@ def PLSA(self,request,context): param_error = True message = 'topic_divider parameter can not be a negative nubmer' - if topic_divider != 0 and num_topics < 2: + if topic_divider == 0 and num_topics < 2: param_error = True message = 'Number of topics should be at least two'