Skip to content

build errors and new version of nseindia.com/api #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alexey-lukashov wants to merge 2 commits into
base: master
Choose a base branch
from
Open

build errors and new version of nseindia.com/api #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5b01e7a
Resolved build errors by adding Apache Commons Lang to project classpath
May 5, 2023
febfdea
Fix API client to work with current NSEIndia, add unit tests, change …
May 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,17 @@ To list available columns, run `!columns` command:

Next, try SQL queries over data

For example:
```shell
!connect jdbc:calcite:model=src/main/resources/model.json admin admin
```
```shell
select "open" from NIFTY_50;
```
```shell
SELECT * FROM "Sectoral"."NIFTY_ENERGY" INNER JOIN "NIFTY_50" ON "Sectoral"."NIFTY_ENERGY"."symbol" = "NIFTY_50"."symbol";
```

### Other Details

* Log File: application.log
31 changes: 21 additions & 10 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>7</source>
<target>7</target>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
<plugin>
Expand Down Expand Up @@ -85,10 +85,20 @@
<properties>
<build.timestamp>${maven.build.timestamp}</build.timestamp>
<calcite.version>1.32.0</calcite.version>
<avatica.version>1.15.0</avatica.version>
<avatica.version>1.22.0</avatica.version>
</properties>

<dependencies>
<dependency>
<groupId>org.htmlunit</groupId>
<artifactId>htmlunit</artifactId>
<version>3.1.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
<dependency>
<groupId>org.apache.calcite.avatica</groupId>
<artifactId>avatica</artifactId>
Expand All @@ -99,15 +109,10 @@
<artifactId>calcite-core</artifactId>
<version>${calcite.version}</version>
</dependency>
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
<version>1.30.2</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.9</version>
<version>2.8.5</version>
</dependency>
<dependency>
<groupId>sqlline</groupId>
Expand All @@ -125,5 +130,11 @@
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.25</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
</project>
103 changes: 40 additions & 63 deletions src/main/java/org/gitcloned/calcite/adapter/nse/NSESchema.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,19 @@
package org.gitcloned.calcite.adapter.nse;

import com.google.common.collect.ImmutableMap;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.calcite.schema.Table;
import org.apache.calcite.schema.impl.AbstractSchema;
import org.gitcloned.nse.NseData;
import org.gitcloned.nse.http.NseHTTP;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.sql.Connection;
import java.util.Locale;
import java.util.Map;

Expand All @@ -23,13 +29,13 @@ public class NSESchema extends AbstractSchema {
private Map<String, Table> tableMap;

private final NseData nse;
private final String MASTER_SCHEMA_URL = "https://www.nseindia.com/api/equity-master";

public NSESchema(String group) {
this.group = group;

NseHTTP nseHttp = new NseHTTP("https://www1.nseindia.com/live_market/dynaContent/live_watch/stock_watch/");

this.nse = new NseData(nseHttp);
NseHTTP nseHttp = new NseHTTP("https://www.nseindia.com", group);
this.nse = new NseData("https://www.nseindia.com/api/equity-stockIndices?index=", nseHttp);
}

public NseData getNseSession() {
Expand Down Expand Up @@ -58,67 +64,38 @@ private Map<String, Table> createTableMap() {
logger.debug(String.format("getting tables for group: '%s'", this.group));

/**
* List all tables under different categories in NSE
* List all tables under different categories in NSE from https://www.nseindia.com/api/equity-master
*/
if (this.group.equals("Broad Market Indices")) {

NSEScannableTable table = (NSEScannableTable) createTable("nifty", "Nifty50");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("juniorNifty", "Nifty Junior");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("niftyMidcap50", "Nifty Midcap 50");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("niftyMidcap150Online", "Nifty Midcap 150");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("niftySmallcap50Online", "Nifty Smlcap 50");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("niftySmallcap250Online", "Nifty Smlcap 250");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);
} else if (this.group.equals("Sectoral Indices")) {

NSEScannableTable table = (NSEScannableTable) createTable("cnxAuto", "Nifty Auto");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("bankNifty", "Nifty Bank");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("cnxEnergy", "Nifty Energy");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("cnxFinance", "Nifty Financial Services");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("cnxFMCG", "Nifty FMCG");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("cnxit", "Nifty IT");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("cnxMetal", "Nifty Metal");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("cnxPharma", "Nifty Pharma");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);

table = (NSEScannableTable) createTable("cnxRealty", "Nifty Realty");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);
} else if (this.group.equals("Thematic Indices")) {

NSEScannableTable table = (NSEScannableTable) createTable("cnxCommodities", "Nifty Commodities");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);
} else if (this.group.equals("Strategy Indices")) {

NSEScannableTable table = (NSEScannableTable) createTable("cnxDividendOppt", "Nifty Dividend Opportunities 50");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);
} else {

NSEScannableTable table = (NSEScannableTable) createTable("sovGold", "Sovereign Gold Bonds");
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);
try {
String masterJson = this.nse.getRequestBuilder().sendGetRequest(MASTER_SCHEMA_URL).getContentAsString();

JsonParser parser = new JsonParser();
JsonElement tree = parser.parse(masterJson);

if (tree.isJsonObject()) {

for(String groupName : ((JsonObject)tree).keySet()) {
if(this.group.equals(groupName)) {
logger.info(String.format("Loading group '%s'", groupName));
JsonElement groupData = ((JsonObject)tree).get(groupName);
if (groupData.isJsonArray()) {
JsonArray group = groupData.getAsJsonArray();
for (JsonElement tableData : group) {
if (tableData.isJsonPrimitive()) {
String tableName = tableData.getAsString();
logger.info(String.format("Loading group '%s' table '%s'", groupName, tableName));

NSEScannableTable table = (NSEScannableTable) createTable(tableName, tableName.replace(" ", "_"));
builder.put(table.getTableName().toUpperCase(Locale.getDefault()), table);
}
}
}
}
}
}
}
catch (IOException e) {
throw new RuntimeException(e);
}

return builder.build();
Expand Down
4 changes: 1 addition & 3 deletions src/main/java/org/gitcloned/calcite/adapter/nse/NSESchemaFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@
public class NSESchemaFactory implements SchemaFactory {

public Schema create(SchemaPlus schemaPlus, String s, Map<String, Object> map) {

NSESchema schema = new NSESchema((String)map.get("group"));
return schema;
return new NSESchema((String)map.get("group"));
}
}
33 changes: 11 additions & 22 deletions src/main/java/org/gitcloned/nse/NseData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,61 +1,50 @@
package org.gitcloned.nse;

import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpResponse;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.gitcloned.nse.http.NseHTTP;
import org.htmlunit.WebResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;

public class NseData {

Logger logger = LoggerFactory.getLogger(NseData.class);

private final NseHTTP requestBuilder;
public String EQUITY_URL;

public NseData(NseHTTP requestBuilder) {
public NseData(String EQUITY_URL, NseHTTP requestBuilder) {
this.EQUITY_URL = EQUITY_URL;
this.requestBuilder = requestBuilder;
}

public NseHTTP getRequestBuilder() {
return requestBuilder;
}

private boolean isSuccessfulResponse (int statusCode) {

if (statusCode - 200 < 0) return false;
if (statusCode - 200 >= 100) return false;
return true;
}

public JsonArray scanTable (String groupName, String tableName) throws IOException, RuntimeException {
WebResponse response = this.requestBuilder.sendGetRequest(EQUITY_URL + URLEncoder.encode(tableName, StandardCharsets.UTF_8.toString()));

StringBuilder api = new StringBuilder("");
api.append(tableName);
api.append("StockWatch.json");

HttpRequest request = this.requestBuilder.buildGetRequest(api.toString(), null);

HttpResponse response = request.execute();

if (!isSuccessfulResponse(response.getStatusCode())) {
throw new RuntimeException("Data Request Failed, status code (" + response.getStatusCode() + "), Response: " + response.parseAsString());
if (!NseHTTP.isSuccessfulResponse(response.getStatusCode())) {
throw new RuntimeException("Data Request Failed, status code (" + response.getStatusCode() + "), Response: " + response.getStatusMessage());
}

String responseString = response.parseAsString();
String responseString = response.getContentAsString();

JsonParser parser = new JsonParser();
JsonElement tree = parser.parse(responseString);

if (tree.isJsonObject()) {

JsonElement rows = tree.getAsJsonObject().get("data");
JsonElement time = tree.getAsJsonObject().get("time");
JsonElement time = tree.getAsJsonObject().get("timestamp");

if (time.isJsonNull()) {
throw new RuntimeException("Data Request Failed: Response is not a valid json, cannot find the 'time' of the data response");
Expand Down
73 changes: 27 additions & 46 deletions src/main/java/org/gitcloned/nse/http/NseHTTP.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,64 +1,45 @@
package org.gitcloned.nse.http;

import com.google.api.client.http.*;
import com.google.api.client.http.javanet.NetHttpTransport;
import org.apache.commons.lang3.StringUtils;
import org.htmlunit.BrowserVersion;
import org.htmlunit.WebClient;
import org.htmlunit.WebResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.security.GeneralSecurityException;
import java.util.Map;

public class NseHTTP {

Logger logger = LoggerFactory.getLogger(NseHTTP.class);

private final String BASE_URL;
private HttpRequestFactory requestFactory;

public NseHTTP(String BASE_URL) {

this.BASE_URL = BASE_URL;
this.requestFactory = new NetHttpTransport().createRequestFactory();

try {

NetHttpTransport.Builder builder = new NetHttpTransport.Builder();
builder.doNotValidateCertificate();

this.requestFactory = builder.build().createRequestFactory();
} catch (GeneralSecurityException e) {
throw new RuntimeException("Got General Security exception: " + e.getMessage());
}
}

private void setHeaders (HttpRequest request, Map<String, String> headers) {
if (headers != null) {

HttpHeaders httpHeaders = request.getHeaders();

for (String key : headers.keySet()) {
httpHeaders.set(key, headers.get(key));
private WebClient webClient;

public NseHTTP(String BASE_URL_FOR_COOKIES, String target) {
this.webClient = new WebClient(BrowserVersion.CHROME);
webClient.getOptions().setJavaScriptEnabled(false);
webClient.getOptions().setCssEnabled(false);
if(StringUtils.isNotBlank(BASE_URL_FOR_COOKIES)) {
logger.info(String.format("Initializing web client cookies from %s for %s", BASE_URL_FOR_COOKIES, target));
try {
WebResponse response = webClient.getPage(BASE_URL_FOR_COOKIES).getWebResponse();
if(200 != response.getStatusCode()) {
throw new RuntimeException(String.format("Can not initialize cookies using %s with result %s, code %s", BASE_URL_FOR_COOKIES, response.getStatusMessage(), response.getStatusCode()));
}
}
catch (IOException e) {
throw new RuntimeException(e);
}
}
}

private void setAuth (HttpRequest request) {

// No auth required
public static boolean isSuccessfulResponse (int statusCode) {
if (statusCode - 200 < 0) return false;
if (statusCode - 200 >= 100) return false;
return true;
}

public HttpRequest buildGetRequest (String api, Map<String, String> headers) throws IOException {

logger.info(String.format("Hitting api '%s'", BASE_URL + api));

HttpRequest request = requestFactory.buildGetRequest(
new GenericUrl(BASE_URL + api));

setHeaders(request, headers);

setAuth(request);

return request;
public WebResponse sendGetRequest (String api) throws IOException {
logger.info(String.format("Hitting api '%s'", api));
return webClient.getPage(api).getWebResponse();
}
}
Loading