Finished with docker file and using supervisor, fix topic divider logic, started circleci integration

.circleci/config.yml

@@ -0,0 +1,48 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: edyirdaw/snet_topic_analysis:v2
+    working_directory: /home/top/dep
+    steps:
+      - checkout
+      - run:
+          name: Run unit tests
+          command: |
+            echo 'starting ...'
+            pwd
+            echo 'unit tests'
+            python3.6 -m grpc_tools.protoc -I. --python_out=.  --grpc_python_out=. service_spec/topic_analysis.proto
+            python3.6 test_topic_analysis_grpc.py
+
+
+  deploy:
+    docker:
+      - image: edyirdaw/snet_topic_analysis:v2
+    steps:
+       - run:
+          name: Run deploy script
+          command: |
+            echo 'starting deploy ...'
+            ssh -o StrictHostKeyChecking=no  $SSH_USER_1@$SSH_HOST_1 "bash /home/eyob/deploy/src_topic.sh"
+
+
+
+workflows:
+  version: 2
+  build-deploy:
+    jobs:
+      - build:
+          filters:
+            branches:
+              only:
+                - master
+                - dev
+      - deploy:
+          requires:
+            - build
+          filters:
+            branches:
+              only:
+                - master
+                - dev

.gitignore

@@ -5,3 +5,4 @@ plsa-service/plsa/_plsa.c
 service_spec/__pycache__/
 service_spec/topic_analysis_pb2.py
 service_spec/topic_analysis_pb2_grpc.py
+Docker/daemons/*.*

Docker/Dockerfile

@@ -36,22 +36,84 @@ RUN sudo pip install -r requirements.txt
 RUN echo 1
 RUN python3.6 -m nltk.downloader all
 
+RUN sudo apt-get install -y supervisor && \
+    sudo rm /etc/supervisor/supervisord.conf
+
+COPY requirements_2.txt /tmp/
+RUN sudo pip install -r requirements_2.txt
+
+RUN sudo mkdir /var/log/kovand/ \
+    /var/log/ropstend/ \
+    /var/log/topic_grpc/ \
+    /var/log/topic_rest/
+
+RUN mkdir /home/top/pay/ \
+    /home/top/daemons/ \
+    /home/top/pay/kovan/ \
+    /home/top/pay/ropsten/ \
+    /home/top/gunicorn/ \
+    /home/top/daemons/kovan/ \
+    /home/top/daemons/ropsten/ \
+    /home/top/dep/ \
+    /home/top/dep/appData/ \
+    /home/top/dep/appData/plsa/ \
+    /home/top/dep/appData/plsa/cleaned/ \
+    /home/top/dep/appData/plsa/dict/ \
+    /home/top/dep/appData/plsa/extracted/ \
+    /home/top/dep/appData/plsa/plsa-parameters
+
+
+ENV DAEMON_VERSION 0.1.6
+
+RUN wget -O snet-daemon.tar.gz https://github.com/singnet/snet-daemon/releases/download/v$DAEMON_VERSION/snet-daemon-v$DAEMON_VERSION-linux-amd64.tar.gz && \
+    tar  --strip-components=1 -xzf snet-daemon.tar.gz && \
+    cp snetd /home/top/daemons/kovan/ && \
+    cp snetd /home/top/daemons/ropsten/
+
+RUN sudo apt-get update && DEBIAN_FRONTEND=noninteractive sudo apt-get install -y locales
+
+RUN sudo sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
+    sudo dpkg-reconfigure --frontend=noninteractive locales && \
+    sudo update-locale LANG=en_US.UTF-8
+
+ENV LANG en_US.UTF-8
+
+RUN echo 32
+
+COPY daemons/snetd_topic_kovan.json /home/top/daemons/kovan/
+COPY daemons/snetd_topic_ropsten.json /home/top/daemons/ropsten/
+COPY gunicorn/config.py /home/top/gunicorn/
+COPY supervisor/kovand.conf /etc/supervisor/conf.d/
+COPY supervisor/ropstend.conf /etc/supervisor/conf.d/
+COPY supervisor/topic_grpc.conf /etc/supervisor/conf.d/
+COPY supervisor/topic_rest.conf /etc/supervisor/conf.d/
+COPY supervisor/supervisord.conf /etc/supervisor/
+
 
-WORKDIR /home/top/
-RUN mkdir dep
 WORKDIR /home/top/dep
 
-#RUN git clone https://github.com/singnet/network-analytics-services.git && \
-#	cd network-analytics-services
 
-RUN whoami
 
-RUN echo 13 && \
-	git clone https://github.com/edyirdaw/topic-analysis.git && cd topic-analysis &&  git checkout --track origin/dev
+# Can be uncommented for local docker building
+
+#RUN whoami
+#
+#RUN echo 13 && \
+#	git clone https://github.com/edyirdaw/topic-analysis.git && cd topic-analysis &&  git checkout --track origin/dev
+#
+#WORKDIR /home/top/dep/topic-analysis
+#
+#RUN pwd
+#RUN python3.6 -m grpc_tools.protoc -I. --python_out=.  --grpc_python_out=. service_spec/topic_analysis.proto
+
+# End of Can be uncommented for local docker building
+
 
-WORKDIR /home/top/dep/topic-analysis
+## Never going to be uncommented
 
-RUN pwd
-RUN python3.6 -m grpc_tools.protoc -I. --python_out=.  --grpc_python_out=. service_spec/topic_analysis.proto
+#CMD ["/usr/bin/supervisord"]
 
+#RUN sudo service supervisor stop
+#RUN sudo service supervisor start
 
+## End of Never going to be uncommented

Docker/gunicorn/config.py

@@ -0,0 +1,10 @@
+workers = 3
+bind = "0.0.0.0:4998"
+errorlog = "/home/top/gunicorn/error.log"
+accesslog = "/home/top/gunicorn/access.log"
+loglevel = "debug"
+max_requests = 10
+pidfile = "/home/top/gunicorn/pid.pid"
+chdir = "/home/top/dep/topic-analysis"
+timeout = 120
+access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" "%(m)s" "%(U)s" "%(q)s" "%(H)s" %(B)s %(T)s %(p)s "%({Header}i)s" "%({Header}o)s" "%({Header}e)s"'

Docker/requirements_2.txt

@@ -0,0 +1 @@
+gunicorn

Docker/supervisor/kovand.conf

@@ -0,0 +1,20 @@
+[supervisord]
+logfile=/dev/null
+nodaemon=true
+
+[program:kovand]
+command=./serve --config /home/top/daemons/kovan/snetd_topic_kovan.json
+directory=/home/top/daemons/kovan/snetd
+user=top
+autostart=true
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+;stderr_logfile=/var/log/kovand/err.log
+;stdout_logfile=/var/log/kovand/out.log
+;redirect_stderr=true
+;stdout_logfile_maxbytes=10MB
+;stdout_logfile_backups=100
+;stderr_logfile_maxbytes=10MB
+;stderr_logfile_backups=100

Docker/supervisor/ropstend.conf

@@ -0,0 +1,20 @@
+[supervisord]
+logfile=/dev/null
+nodaemon=true
+
+[program:ropstend]
+command=./serve --config /home/top/daemons/ropsten/snetd_topic_ropsten.json
+directory=/home/top/daemons/ropsten/snetd
+user=top
+autostart=true
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+;stderr_logfile=/var/log/ropstend/err.log
+;stdout_logfile=/var/log/ropstend/out.log
+;redirect_stderr=true
+;stdout_logfile_maxbytes=10MB
+;stdout_logfile_backups=100
+;stderr_logfile_maxbytes=10MB
+;stderr_logfile_backups=100

Docker/supervisor/supervisord.conf

@@ -0,0 +1,30 @@
+; supervisor config file
+
+[unix_http_server]
+file=/var/run/supervisor.sock   ; (the path to the socket file)
+chmod=0700                       ; sockef file mode (default 0700)
+
+[supervisord]
+logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
+pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
+childlogdir=/var/log/supervisor            ; ('AUTO' child log dir, default $TEMP)
+;logfile_maxbytes=10MB
+;logfile_backups=10
+
+; the below section must remain in the config file for RPC
+; (supervisorctl/web interface) to work, additional interfaces may be
+; added by defining them in separate rpcinterface: sections
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL  for a unix socket
+
+; The [include] section can just contain the "files" setting.  This
+; setting can list multiple files (separated by whitespace or
+; newlines).  It can also contain wildcards.  The filenames are
+; interpreted as relative to this file.  Included files *cannot*
+; include files themselves.
+
+[include]
+files = /etc/supervisor/conf.d/*.conf

Docker/supervisor/topic_grpc.conf

@@ -0,0 +1,20 @@
+[supervisord]
+logfile=/dev/null
+nodaemon=true
+
+[program:topic_grpc]
+command=python3.6 topic_analysis_grpc.py
+directory=/home/top/dep/topic-analysis/
+user=top
+autostart=true
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+;stderr_logfile=/var/log/topic_grpc/err.log
+;stdout_logfile=/var/log/topic_grpc/out.log
+;redirect_stderr=true
+;stdout_logfile_maxbytes=10MB
+;stdout_logfile_backups=100
+;stderr_logfile_maxbytes=10MB
+;stderr_logfile_backups=100

Docker/supervisor/topic_rest.conf

@@ -0,0 +1,20 @@
+[supervisord]
+logfile=/dev/null
+nodaemon=true
+
+[program:topic_rest]
+command=gunicorn -c config.py analysis_results:app
+directory=/home/top/gunicorn/
+user=top
+autostart=true
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+redirect_stderr=true
+;stderr_logfile=/var/log/topic_rest/err.log
+;stdout_logfile=/var/log/topic_rest/out.log
+;redirect_stderr=true
+;stdout_logfile_maxbytes=10MB
+;stdout_logfile_backups=100
+;stderr_logfile_maxbytes=10MB
+;stderr_logfile_backups=100

analysis_results.py

@@ -137,4 +137,4 @@ def not_found(error):
 
 
     # app.run(debug=True)
-    app.run(debug=False,port=4999)
+    app.run(debug=False,port=4998)

snet_test_client.py

@@ -30,6 +30,20 @@ def sample_data():
 
     return docs
 
+def sample_data_2():
+
+    path = str(pathlib.Path(os.path.abspath('')).parents[0])+'/topic-analysis/docs/tests/test_doc.txt'
+
+    docs = []
+
+    with open(path, "r") as read_file:
+        text = read_file.read().splitlines()
+
+    for k in text:
+        if k != "":
+            docs.append(k)
+
+    return docs
 
 def csv_reader():
 
@@ -58,7 +72,7 @@ def csv_reader():
 
 
 def try_plsa():
-    channel = grpc.insecure_channel('localhost:5000')
+    channel = grpc.insecure_channel('localhost:5001')
     # channel = grpc.insecure_channel('172.17.0.75:5001')
     stub = topic_analysis_pb2_grpc.TopicAnalysisStub(channel)
 
@@ -75,6 +89,23 @@ def try_plsa():
     print(resp.handle)
 
 
+def try_plsa_2():
+    # channel = grpc.insecure_channel('localhost:5000')
+    channel = grpc.insecure_channel('172.17.1.53:5001')
+    # channel = grpc.insecure_channel('172.17.0.75:5001')
+    stub = topic_analysis_pb2_grpc.TopicAnalysisStub(channel)
+
+
+    plsa_request = topic_analysis_pb2.PLSARequest(docs=sample_data_2(),num_topics=2,maxiter=22,beta=1)
+
+    resp = stub.PLSA(plsa_request)
+
+
+    print(resp.status)
+    print('^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^')
+    print(resp.message)
+    print('^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^')
+    print(resp.handle)
 
 
 
@@ -83,7 +114,8 @@ def try_plsa():
 
 if __name__ == '__main__':
 
-    try_plsa()
+    # try_plsa()
+    try_plsa_2()
     # csv_reader()
 
 

test_topic_analysis_grpc.py

@@ -23,7 +23,7 @@ def setUp(self):
         self.app = analysis_results.app.test_client()
         self.docs = []
 
-        sample_doc = 'docs/test_doc.txt'
+        sample_doc = 'docs/tests/test_doc.txt'
         with open(sample_doc,'r') as f:
             self.docs = f.read().splitlines()
 

topic_analysis_grpc.py

@@ -55,7 +55,7 @@ def PLSA(self,request,context):
             param_error = True
             message = 'topic_divider parameter can not be a negative nubmer'
 
-        if topic_divider != 0 and num_topics < 2:
+        if topic_divider == 0 and num_topics < 2:
             param_error = True
             message = 'Number of topics should be at least two'