Changes and upgrades from HDX

README.rst

@@ -1,3 +1,7 @@
+OCHA Fork
++++++++++
+Security protocols at the UN require all services be run on UN servers, so dataproxy had to be modified to run indepedent of Google App Engine.
+
 Data Proxy
 ++++++++++
 

dataproxy/app.py

@@ -95,6 +95,8 @@ def __call__(self, environ, start_response):
         flow['app']['config'] = AttributeDict()
         flow['app']['config']['proxy'] = AttributeDict(max_length=int(self.max_length))
         flow['environ'] = environ
+        if not 'HTTP_COOKIE' in flow.environ:
+            flow['environ']['HTTP_COOKIE'] = ''
         flow['http_response'] = HTTPResponseMarble()
         flow.http_response.header_list = [
             dict(name='Content-Type', value='application/javascript'),
@@ -128,7 +130,12 @@ def __call__(self, environ, start_response):
             return error(title=title, message=msg)
 
     def index(self, flow):
-        if not flow.query.has_key('url'):
+        if not self.from_ckan(flow.environ['HTTP_COOKIE']):
+            title = 'ckan only'
+            msg = 'Dataproxy only accepts requests from CKAN installations'
+            flow.http_response.status = '200 Error %s'%title
+            flow.http_response.body = error(title=title, message=msg)
+        elif not flow.query.has_key('url'):
             title = 'url query parameter missing'
             msg = 'Please read the dataproxy API format documentation: https://github.com/okfn/dataproxy'
             flow.http_response.status = '200 Error %s'%title
@@ -142,6 +149,14 @@ def index(self, flow):
                 flow.http_response.status = '200 %s %s' % (e.error, e.title)
                 flow.http_response.body = error(title=e.title, message=e.message)
 
+    def from_ckan(self,cookies):
+        #Checks to see if there's a ckan cookie present
+        #for c in cookies.split(';'):
+        #    t = c.split('=')
+        #    if t[0].strip().lower() == 'ckan':
+        #        return True
+        #return False
+        return True
 
     def proxy_query(self, flow, url, query):
         parts = urlparse.urlparse(url)
@@ -221,15 +236,15 @@ def transform(type_name, flow, url, query, max_results):
     if 'encoding' in query:
         encoding = query["encoding"].value
     if type_name == 'csv':
-        stream = urllib2.urlopen(url)
+        stream = create_stream(flow, url)
         records, metadata = dataconverters.commas.parse(stream, encoding=encoding,
                 window=window, guess_types=guess_types)
     elif type_name == 'tsv':
-        stream = urllib2.urlopen(url)
+        stream = create_stream(flow, url)
         records, metadata = dataconverters.commas.parse(stream, delimiter='\t',
                 encoding=encoding, window=window, guess_types=guess_types)
     elif type_name == 'xls' or type_name == 'xlsx':
-        stream = urllib2.urlopen(url)
+        stream = create_stream(flow, url)
         length = int(stream.headers.get('content-length', 0))
         # max_length = flow.app.config.proxy.max_length
         max_length = 5000000 # ~ 5Mb
@@ -243,12 +258,37 @@ def transform(type_name, flow, url, query, max_results):
         else:
             sheet_number = 1
         records, metadata = dataconverters.xls.parse(stream,
-                excel_type=type_name, worksheet=sheet_number,
+                excel_type=type_name,
                 guess_types=guess_types)
     else:
         raise Exception("Resource type not supported '%s'" % type_name)
     return (records, metadata)
 
+def from_hdx(url):
+    url = urlparse.urlparse(url)
+    url = url.netloc.split(':')
+    if url[0][-14:] == 'hdx.rwlabs.org':
+        return True
+    else:
+        return False
+
+def create_stream(flow, url):
+    hdx = from_hdx(url)
+    if hdx:
+        request = urllib2.Request(url, headers={"Cookie" : flow.environ['HTTP_COOKIE']})
+        try:
+            stream = urllib2.urlopen(request)
+        except urllib2.HTTPError, err:
+            if err.code == 403:
+                raise Exception("You do not have permission to access this file.")
+            else:
+                if hdx:
+                    raise Exception("File could not be fetched from filestore.")
+                else:
+                    raise Exception("File could not be fetched.")
+    else:
+        stream = urllib2.urlopen(url)
+    return stream
 
 if __name__ == '__main__':
     from wsgiref.util import setup_testing_defaults

dataproxy/dataproxy.py

@@ -1,11 +1,36 @@
-from google.appengine.ext.webapp.util import run_wsgi_app
-import os
+#from google.appengine.ext.webapp.util import run_wsgi_app
+import os, sys
 from app import JsonpDataProxy
 
 application = JsonpDataProxy(3000000)
 
+
+def _start_response(status, headers, exc_info=None):
+	if exc_info is not None:
+		raise exc_info[0], exc_info[1], exc_info[2]
+	print "Status: %s" % status
+	for name, val in headers:
+		print "%s: %s" % (name, val)
+	print
+	return sys.stdout.write
+
 def main():
-    run_wsgi_app(application)
+    env = dict(os.environ)
+    env["wsgi.input"] = sys.stdin
+    env["wsgi.errors"] = sys.stderr
+    env["wsgi.version"] = (1,0)
+    env["wsgi.run_once"] = True
+    env["wsgi.url_scheme"] = wsgiref.util.guess_scheme(env)
+    env["msgi.multithread"] = False
+    env["wsgi.multiprocess"] = False
+    result = application(env, _start_response)
+    try:
+    	if result is not None:
+    		for data in result:
+    			sys.stdout.write(data)
+    finally:
+    	if hasattr(result, 'close'):
+    		result.close()
 
 if __name__ == "__main__":
     main()