Shift to only one kv read per read-request on unofficial sources. Sta…

…rt caching full responses instead of parsing and assembling jsons on the fly
amodm · Mar 19, 2020 · b2a5814c2398a3ad5b224b4cf53fba371146330c · b2a5814
1 parent b97221f
commit b2a5814c2398a3ad5b224b4cf53fba371146330c
Unified Split

Showing with 22 additions and 38 deletions.

+6 −11 api.js

+0 −3 constants.js

+8 −2 data/unofficial/index.js

+5 −3 index.js

+1 −1 refresh-hospital-beds.js

+2 −18 unofficial.js
diff --git a/api.js b/api.js
@@ -1,21 +1,16 @@
-import { Store } from './store';
-import { STORE_KEYS } from './constants';
-
-export async function fetchTimestamps() {
-    return Promise.all([Store.get(STORE_KEYS.LAST_REFRESHED), Store.get(STORE_KEYS.LAST_UPDATED_ORIGIN)]);
-}
-
+/**
+ * Returns the raw response without any modification, treating @data as the body
+ */
 export function rawResponse(data) {
     return new Response(typeof data === 'string' ? data : JSON.stringify(data), { headers: standardHeaders });
 }
 
-export async function successResponse(data, timestampsPromise) {
-    const timestamps = await timestampsPromise;
+export function successResponse(data) {
     const output = {
         'success': true,
         'data': data,
-        'lastRefreshed': new Date(parseInt(timestamps[0])).toISOString(),
-        'lastOriginUpdate': new Date(parseInt(timestamps[1])).toISOString()
+        'lastRefreshed': new Date().toISOString(),
+        'lastOriginUpdate': new Date().toISOString()
     };
     return new Response(JSON.stringify(output), { headers: standardHeaders });
 }

diff --git a/constants.js b/constants.js
@@ -1,8 +1,5 @@
 export const STORE_KEYS = {
-    LAST_UPDATED_ORIGIN: "last_updated_origin",
-    LAST_REFRESHED: "last_refreshed",
     CASE_COUNTS: "case_counts",
-    UNOFFICIAL_SRC_PREFIX: "unofficial_src_",
 
     /* All the following keys represent fully cached responses, with no need for assembly or parsing of json */
     CACHED_CASE_COUNTS: "cached_case_counts",

diff --git a/data/unofficial/index.js b/data/unofficial/index.js
@@ -71,10 +71,16 @@ async function updateStatewiseDataFromCovid19IndiaOrg() {
  * Update the unofficial source record in Workers KV
  */
 async function updateUnofficialSource(sourceId, data, suffix=undefined) {
-    const finalData = {source: sourceId, lastRefreshed: new Date().toISOString(), ...data};
+    const current = new Date().toISOString();
+    const finalData = {
+        success: true,
+        data: {source: sourceId, lastRefreshed: current, ...data},
+        lastRefreshed: current,
+        lastOriginUpdate: current
+    };
     const accountId = process.env['CF_ACCOUNT_ID'];
     const namespaceId = process.env['CF_NAMESPACE_ID'];
-    const key = suffix ? `unofficial_src_${sourceId}_${suffix}` : `unofficial_src_${sourceId}`;
+    const key = suffix ? `cached_unofficial_src_${sourceId}_${suffix}` : `cached_unofficial_src_${sourceId}`;
     const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/values/${key}`;
     await fetch(url, {
         method: 'PUT',

diff --git a/index.js b/index.js
@@ -1,5 +1,5 @@
 import { refreshAllOfficialSources } from "./refresh";
-import { getAllUnofficialSources, getUnofficialSource } from "./unofficial";
+import { getAllUnofficialSources } from "./unofficial";
 import {errorResponse, rawResponse} from "./api";
 import { Store } from "./store";
 import { STORE_KEYS } from "./constants";
@@ -36,7 +36,9 @@ const routeHandlers = {
   '/stats/hospitals': () => cachedData(STORE_KEYS.CACHED_HOSPITAL_BEDS_COUNT),
   '/notifications': () => cachedData(STORE_KEYS.CACHED_NOTIFICATIONS),
   '/unofficial/sources': getAllUnofficialSources,
-  '/unofficial/covid19india.org': getUnofficialSource,
-  '/unofficial/covid19india.org/statewise': getUnofficialSource,
+  '/unofficial/covid19india.org': () =>
+      cachedData(STORE_KEYS.CACHED_UNOFFICIAL_SRC_PREFIX + "covid19india.org"),
+  '/unofficial/covid19india.org/statewise': () =>
+      cachedData(STORE_KEYS.CACHED_UNOFFICIAL_SRC_PREFIX + "covid19india.org_statewise"),
   '/refresh': refreshAllOfficialSources
 };
diff --git a/refresh-hospital-beds.js b/refresh-hospital-beds.js
@@ -1,4 +1,4 @@
-import {errorResponse, rawResponse, successResponse} from './api';
+import { errorResponse, rawResponse } from './api';
 import { Store } from './store';
 import { STORE_KEYS } from './constants';
 

diff --git a/unofficial.js b/unofficial.js
@@ -1,26 +1,10 @@
-import { Store } from "./store";
-import { STORE_KEYS } from "./constants";
-import { errorResponse, fetchTimestamps, successResponse } from "./api";
+import { successResponse } from "./api";
 
 /**
  * Get all unofficial sources
  */
 export async function getAllUnofficialSources() {
-    const tsPromise = fetchTimestamps();
-    return successResponse({ "unofficialSources": ALL_UNOFFICIAL_SOURCES }, tsPromise);
-}
-
-export async function getUnofficialSource(request, path) {
-    const tsPromise = fetchTimestamps();
-    const arr = path.split('/');
-    if (arr.length < 3 || !ALL_UNOFFICIAL_SOURCES.includes(arr[2])) {
-        return errorResponse({ reason: "invalid source" }, tsPromise, 404);
-    }
-    else {
-        const suffix = arr.length > 3 ? `_${arr[3]}` : '';
-        const data = await Store.get(STORE_KEYS.UNOFFICIAL_SRC_PREFIX + arr[2] + suffix);
-        return successResponse(JSON.parse(data), tsPromise);
-    }
+    return successResponse({ "unofficialSources": ALL_UNOFFICIAL_SOURCES });
 }
 
 const COVID19INDIA_ORG = 'covid19india.org';