pull develop into feature description

.htaccess

@@ -10,10 +10,9 @@ RewriteRule ^(.*)$ https://www.openml.org/api_new/$1 [L,P]
 RewriteCond $1 !^(questions|SWF|img|docs|downloads|GFX|favicon\.ico|tiny_mce|index\.php|js|css|robots\.txt)
 RewriteRule ^(.*)$ index.php/$1 [L]
 
-
+RedirectMatch 301 ^/?$ http://api.openml.org/home
 
 <IfModule mod_headers.c>
-   SetEnvIfNoCase Origin "https?://(www\.)?(openml\.org|staging\.openml\.org)(:\d+)?$" ACAO=$0
-   Header set Access-Control-Allow-Origin %{ACAO}e env=ACAO
+   Header set Access-Control-Allow-Origin "*"
    Header set Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept"
 </IfModule>

openml_OS/libraries/ElasticSearch.php

@@ -338,7 +338,7 @@ public function index($type, $id = false, $altmetrics=True, $verbosity=0) {
         }
     }
 
-    public function index_from($type, $id = false, $verbosity=1, $altmetrics=True) {
+    public function index_from($type, $id = false, $verbosity=1, $altmetrics=False) {
         //bootstrap
         $indexParams['index'] = $type;
         if(! $this->client->indices()->getMapping($indexParams))
@@ -1340,11 +1340,11 @@ public function index_run($id, $start_id = 0, $altmetrics=True, $verbosity=0) {
               $submitted += sizeof($responses['items']);
               if ($verbosity) {
                 #echo "-  completed ".str_pad($submitted, 9, ' ', STR_PAD_RIGHT);
-                echo "\033[31D";
+                #echo "\033[31D";
               }
            }
          } elseif($verbosity) {
-           echo "\033[9D";
+           #echo "\033[9D";
          }
          $rid += $incr;
        }
@@ -1919,7 +1919,10 @@ public function index_data($id, $start_id = 0, $altmetrics=True, $verbosity=0) {
 
     private function build_data($d, $altmetrics=True) {
         $description_record = $this->CI->Dataset_description->getWhereSingle('did =' . $d->did, 'version DESC');
-        $headless_description = trim(preg_replace('/\s+/', ' ', preg_replace('/^\*{2,}.*/m', '', $description_record->description)));
+        if(!$description_record){
+           return 'Could not find description of dataset ' . $d->did;
+        }
+	$headless_description = trim(preg_replace('/\s+/', ' ', preg_replace('/^\*{2,}.*/m', '', $description_record->description)));
         $new_data = array(
             'data_id' => $d->did,
             'name' => $d->name,

openml_OS/models/api/v1/Api_data.php

@@ -502,22 +502,28 @@ private function data_fork() {
     $latest_version = $this->Dataset-> getWhereSingle('`name` = "' . $dataset->name . '"', 'CAST(`version` AS DECIMAL) DESC');
     $dataset->version = $latest_version->version + 1;
     unset($dataset->did);
-    $data_id = $this->Dataset->insert($dataset);
-    if (!$data_id) {
+    $new_data_id = $this->Dataset->insert($dataset);
+    if (!$new_data_id) {
       $this->returnError(1072, $this->version);
       return;
     }
+
+    // create a copy of the latest description
+    $description_record = $this->Dataset_description->getWhereSingle('did =' . $data_id, 'version DESC');
+    $description_record->did = $new_data_id;
+    $description_record->version = "1";
+    $this->Dataset_description->insert($description_record);
 
     // update elastic search index.  
     try {
-      $this->elasticsearch->index('data', $data_id);
+      $this->elasticsearch->index('data', $new_data_id);
     } catch (Exception $e) {
       $this->returnError(105, $this->version, $this->openmlGeneralErrorCode, $e->getMessage());
       return;
     }
 
     // Return data id, for user to verify changes
-    $this->xmlContents( 'data-fork', $this->version, array( 'id' => $data_id) );
+    $this->xmlContents( 'data-fork', $this->version, array( 'id' => $new_data_id) );
   }
 
   private function data_edit() {
@@ -757,9 +763,11 @@ private function data($data_id) {
     if ($data_status != false) {
       $dataset->status = $data_status->status;
     }
-
-    $dataset->minio_url = 'http://openml1.win.tue.nl/dataset' . $data_id . '/dataset_' . $data_id . '.pq';
-    $this->xmlContents( 'data-get', $this->version, $dataset );
+    if ($dataset->format != 'Sparse_ARFF') {
+      $dataset->parquet_url = 'http://openml1.win.tue.nl/dataset' . $data_id . '/dataset_' . $data_id . '.pq';      
+      $dataset->minio_url = 'http://openml1.win.tue.nl/dataset' . $data_id . '/dataset_' . $data_id . '.pq';
+    }
+      $this->xmlContents( 'data-get', $this->version, $dataset );
   }
 
   private function data_reset($data_id) {

openml_OS/views/pages/api_new/v1/xml/data-get.tpl.php

@@ -12,6 +12,7 @@
   <?php if ($language != null): ?><oml:language><?php echo htmlspecialchars($language); ?></oml:language><?php endif; ?>
   <?php if ($licence != null): ?><oml:licence><?php echo htmlspecialchars($licence); ?></oml:licence><?php endif; ?>
   <oml:url><?php echo htmlspecialchars($url); ?></oml:url>
+  <?php if ($parquet_url != null): ?><oml:parquet_url><?php echo htmlspecialchars($parquet_url); ?></oml:parquet_url><?php endif; ?>
   <?php if ($file_id != null): ?><oml:file_id><?php echo $file_id; ?></oml:file_id><?php endif; ?>
   <?php if ($default_target_attribute != null): ?><oml:default_target_attribute><?php echo htmlspecialchars($default_target_attribute); ?></oml:default_target_attribute><?php endif; ?>
   <?php if ($row_id_attribute != null): ?><oml:row_id_attribute><?php echo htmlspecialchars($row_id_attribute); ?></oml:row_id_attribute><?php endif; ?>