From 3348c4991c211eceef2099b5ce3483daad7b9076 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Tue, 17 Dec 2019 21:14:35 +0100 Subject: [PATCH 1/7] fix: initial timestamp from/to, search max settings.SCIDASH_INITIAL_SEARCH_QUARTERS quarters back, if not found show settings.SCIDASH_INITIAL_SEARCH_QUARTERS quarters back period --- scidash/main/settings.py | 4 ++++ scidash/sciunittests/views.py | 12 +++--------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/scidash/main/settings.py b/scidash/main/settings.py index 9d0b0b9c..0fd219bc 100644 --- a/scidash/main/settings.py +++ b/scidash/main/settings.py @@ -249,3 +249,7 @@ CELERY_RESULT_BACKEND = 'django-db' NO_IMPORT_TAG = 'unschedulable' + +# SCIDASH +# Initial search number of quarters to search back in time. +SCIDASH_INITIAL_SEARCH_QUARTERS = 12 diff --git a/scidash/sciunittests/views.py b/scidash/sciunittests/views.py index a70d419c..99e91b50 100644 --- a/scidash/sciunittests/views.py +++ b/scidash/sciunittests/views.py @@ -17,16 +17,10 @@ def get(self, request, *args, **kwargs): day=current_date.day ) - three_month_ago = current_date_iso - three_month_period - six_month_ago = three_month_ago - three_month_period - nine_month_ago = six_month_ago - three_month_period - tvelwe_month_ago = nine_month_ago - three_month_period - acceptable_period = None - for period in [ - three_month_ago, six_month_ago, nine_month_ago, tvelwe_month_ago - ]: + for quarter in range(1, s.SCIDASH_INITIAL_SEARCH_QUARTERS + 1): + period = current_date_iso - quarter*three_month_period count = ScoreInstance.objects.filter( timestamp__gte=period, timestamp__lt=current_date_iso ).count() @@ -36,7 +30,7 @@ def get(self, request, *args, **kwargs): break if acceptable_period is None: - acceptable_period = tvelwe_month_ago + acceptable_period = period return Response( { From 4cdc0abca64b48928a321d59b3b7eb2aa39ffd51 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 19 Dec 2019 12:49:52 +0100 Subject: [PATCH 2/7] feat: add index on scoreinstances(timestamp) --- scidash/sciunittests/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scidash/sciunittests/models.py b/scidash/sciunittests/models.py index 2f17f852..f3094fb1 100644 --- a/scidash/sciunittests/models.py +++ b/scidash/sciunittests/models.py @@ -243,6 +243,9 @@ def prediction(self): class Meta: ordering = ['-timestamp'] + indexes = [ + models.Index(fields=['-timestamp',]), + ] def __str__(self): return "Score for {0} in {1} test instance".format( From 1138ba34af5910cf341bad403400599ce87e8f62 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 12 Dec 2019 10:36:38 -0500 Subject: [PATCH 3/7] fix: add missing migration to git repo --- .../migrations/0046_auto_20191211_0156.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 scidash/sciunittests/migrations/0046_auto_20191211_0156.py diff --git a/scidash/sciunittests/migrations/0046_auto_20191211_0156.py b/scidash/sciunittests/migrations/0046_auto_20191211_0156.py new file mode 100644 index 00000000..7e28b229 --- /dev/null +++ b/scidash/sciunittests/migrations/0046_auto_20191211_0156.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2019-12-11 01:56 +from __future__ import unicode_literals + +import datetime +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import scidash.sciunittests.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sciunittests', '0045_auto_20190712_1041'), + ] + + operations = [ + migrations.AddField( + model_name='testclass', + name='default_params', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, encoder=scidash.sciunittests.models.JSONEncoder, null=True), + ), + migrations.AlterField( + model_name='scoreinstance', + name='timestamp', + field=models.DateTimeField(default=datetime.datetime.today), + ), + migrations.AlterField( + model_name='testinstance', + name='timestamp', + field=models.DateTimeField(auto_now=True), + ), + migrations.AlterField( + model_name='testsuite', + name='timestamp', + field=models.DateTimeField(default=datetime.datetime.today), + ), + ] From abfdda4ed81f0b7d564a7ba3303b4bd8c844488c Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 12 Dec 2019 15:09:03 -0500 Subject: [PATCH 4/7] fix: renames 0046_auto_20191211_0156.py -> 0046_auto_20190827_0221.py --- .../{0046_auto_20191211_0156.py => 0046_auto_20190827_0221.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scidash/sciunittests/migrations/{0046_auto_20191211_0156.py => 0046_auto_20190827_0221.py} (100%) diff --git a/scidash/sciunittests/migrations/0046_auto_20191211_0156.py b/scidash/sciunittests/migrations/0046_auto_20190827_0221.py similarity index 100% rename from scidash/sciunittests/migrations/0046_auto_20191211_0156.py rename to scidash/sciunittests/migrations/0046_auto_20190827_0221.py From d28559bea9f90a3c46b3fc974c0ca033d40d98aa Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Thu, 19 Dec 2019 13:23:18 +0100 Subject: [PATCH 5/7] feat: added index on scoreinstance.timestamp desc --- .../migrations/0047_auto_20191219_1217.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 scidash/sciunittests/migrations/0047_auto_20191219_1217.py diff --git a/scidash/sciunittests/migrations/0047_auto_20191219_1217.py b/scidash/sciunittests/migrations/0047_auto_20191219_1217.py new file mode 100644 index 00000000..3d8f5f1b --- /dev/null +++ b/scidash/sciunittests/migrations/0047_auto_20191219_1217.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.7 on 2019-12-19 12:17 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sciunittests', '0046_auto_20190827_0221'), + ] + + operations = [ + migrations.AddIndex( + model_name='scoreinstance', + index=models.Index(fields=['-timestamp'], name='sciunittest_timesta_aa7614_idx'), + ), + ] From 479f55e56efe06609b41a545f50c6d7234100f94 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Fri, 20 Dec 2019 10:09:20 +0100 Subject: [PATCH 6/7] feat: return acceptable_period based on top N scores --- scidash/sciunittests/views.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/scidash/sciunittests/views.py b/scidash/sciunittests/views.py index 99e91b50..150e8f88 100644 --- a/scidash/sciunittests/views.py +++ b/scidash/sciunittests/views.py @@ -9,7 +9,21 @@ class DateRangeView(APIView): def get(self, request, *args, **kwargs): - three_month_period = datetime.timedelta(3 * 365 / 12) + """Returns the initial search period (acceptable_period). + for the first N (settings.ACCEPTABLE_SCORE_INSTANCES_AMOUNT) scores. + + Parameters + ---------- + - + + Returns + ------- + JSON object + { + "current_date": "", + "acceptable_period": "" + } + """ current_date = datetime.date.today() + datetime.timedelta(days=1) current_date_iso = datetime.datetime( year=current_date.year, @@ -17,20 +31,11 @@ def get(self, request, *args, **kwargs): day=current_date.day ) - acceptable_period = None - - for quarter in range(1, s.SCIDASH_INITIAL_SEARCH_QUARTERS + 1): - period = current_date_iso - quarter*three_month_period - count = ScoreInstance.objects.filter( - timestamp__gte=period, timestamp__lt=current_date_iso - ).count() - - if count > s.ACCEPTABLE_SCORE_INSTANCES_AMOUNT: - acceptable_period = period - break - - if acceptable_period is None: - acceptable_period = period + scores = ScoreInstance.objects.filter(timestamp__lt=current_date_iso)[:s.ACCEPTABLE_SCORE_INSTANCES_AMOUNT] + if scores: + acceptable_period = scores[len(scores)-1].timestamp + else: + acceptable_period = current_date_iso - datetime.date.year return Response( { From c42955703a54ad100f407f7796dbf371f7eeff75 Mon Sep 17 00:00:00 2001 From: Zoran Sinnema Date: Fri, 20 Dec 2019 10:31:11 +0100 Subject: [PATCH 7/7] fix: rework acceptable_period calculation --- scidash/sciunittests/views.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scidash/sciunittests/views.py b/scidash/sciunittests/views.py index 150e8f88..e3109d9d 100644 --- a/scidash/sciunittests/views.py +++ b/scidash/sciunittests/views.py @@ -31,11 +31,20 @@ def get(self, request, *args, **kwargs): day=current_date.day ) - scores = ScoreInstance.objects.filter(timestamp__lt=current_date_iso)[:s.ACCEPTABLE_SCORE_INSTANCES_AMOUNT] + scores = ScoreInstance.objects.filter( + timestamp__lt=current_date_iso).order_by('-timestamp') \ + [:s.ACCEPTABLE_SCORE_INSTANCES_AMOUNT] if scores: - acceptable_period = scores[len(scores)-1].timestamp + # found scores, acceptable period is scores last.timestamp + # because sorting is DESC timestamp + acceptable_period = scores.reverse()[0].timestamp else: - acceptable_period = current_date_iso - datetime.date.year + # acceptable period defaults to current date - 1 year + acceptable_period = datetime.datetime( + year=current_date.year-1, + month=current_date.month, + day=current_date.day + ) return Response( {