Skip to content
Permalink
Browse files

Shift to only one kv read per read-request on unofficial sources. Sta…

…rt caching full responses instead of parsing and assembling jsons on the fly
  • Loading branch information
amodm committed Mar 19, 2020
1 parent b97221f commit b2a5814c2398a3ad5b224b4cf53fba371146330c
Showing with 22 additions and 38 deletions.
  1. +6 −11 api.js
  2. +0 −3 constants.js
  3. +8 −2 data/unofficial/index.js
  4. +5 −3 index.js
  5. +1 −1 refresh-hospital-beds.js
  6. +2 −18 unofficial.js
17 api.js
@@ -1,21 +1,16 @@
import { Store } from './store';
import { STORE_KEYS } from './constants';

export async function fetchTimestamps() {
return Promise.all([Store.get(STORE_KEYS.LAST_REFRESHED), Store.get(STORE_KEYS.LAST_UPDATED_ORIGIN)]);
}

/**
* Returns the raw response without any modification, treating @data as the body
*/
export function rawResponse(data) {
return new Response(typeof data === 'string' ? data : JSON.stringify(data), { headers: standardHeaders });
}

export async function successResponse(data, timestampsPromise) {
const timestamps = await timestampsPromise;
export function successResponse(data) {
const output = {
'success': true,
'data': data,
'lastRefreshed': new Date(parseInt(timestamps[0])).toISOString(),
'lastOriginUpdate': new Date(parseInt(timestamps[1])).toISOString()
'lastRefreshed': new Date().toISOString(),
'lastOriginUpdate': new Date().toISOString()
};
return new Response(JSON.stringify(output), { headers: standardHeaders });
}
@@ -1,8 +1,5 @@
export const STORE_KEYS = {
LAST_UPDATED_ORIGIN: "last_updated_origin",
LAST_REFRESHED: "last_refreshed",
CASE_COUNTS: "case_counts",
UNOFFICIAL_SRC_PREFIX: "unofficial_src_",

/* All the following keys represent fully cached responses, with no need for assembly or parsing of json */
CACHED_CASE_COUNTS: "cached_case_counts",
@@ -71,10 +71,16 @@ async function updateStatewiseDataFromCovid19IndiaOrg() {
* Update the unofficial source record in Workers KV
*/
async function updateUnofficialSource(sourceId, data, suffix=undefined) {
const finalData = {source: sourceId, lastRefreshed: new Date().toISOString(), ...data};
const current = new Date().toISOString();
const finalData = {
success: true,
data: {source: sourceId, lastRefreshed: current, ...data},
lastRefreshed: current,
lastOriginUpdate: current
};
const accountId = process.env['CF_ACCOUNT_ID'];
const namespaceId = process.env['CF_NAMESPACE_ID'];
const key = suffix ? `unofficial_src_${sourceId}_${suffix}` : `unofficial_src_${sourceId}`;
const key = suffix ? `cached_unofficial_src_${sourceId}_${suffix}` : `cached_unofficial_src_${sourceId}`;
const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/storage/kv/namespaces/${namespaceId}/values/${key}`;
await fetch(url, {
method: 'PUT',
@@ -1,5 +1,5 @@
import { refreshAllOfficialSources } from "./refresh";
import { getAllUnofficialSources, getUnofficialSource } from "./unofficial";
import { getAllUnofficialSources } from "./unofficial";
import {errorResponse, rawResponse} from "./api";
import { Store } from "./store";
import { STORE_KEYS } from "./constants";
@@ -36,7 +36,9 @@ const routeHandlers = {
'/stats/hospitals': () => cachedData(STORE_KEYS.CACHED_HOSPITAL_BEDS_COUNT),
'/notifications': () => cachedData(STORE_KEYS.CACHED_NOTIFICATIONS),
'/unofficial/sources': getAllUnofficialSources,
'/unofficial/covid19india.org': getUnofficialSource,
'/unofficial/covid19india.org/statewise': getUnofficialSource,
'/unofficial/covid19india.org': () =>
cachedData(STORE_KEYS.CACHED_UNOFFICIAL_SRC_PREFIX + "covid19india.org"),
'/unofficial/covid19india.org/statewise': () =>
cachedData(STORE_KEYS.CACHED_UNOFFICIAL_SRC_PREFIX + "covid19india.org_statewise"),
'/refresh': refreshAllOfficialSources
};
@@ -1,4 +1,4 @@
import {errorResponse, rawResponse, successResponse} from './api';
import { errorResponse, rawResponse } from './api';
import { Store } from './store';
import { STORE_KEYS } from './constants';

@@ -1,26 +1,10 @@
import { Store } from "./store";
import { STORE_KEYS } from "./constants";
import { errorResponse, fetchTimestamps, successResponse } from "./api";
import { successResponse } from "./api";

/**
* Get all unofficial sources
*/
export async function getAllUnofficialSources() {
const tsPromise = fetchTimestamps();
return successResponse({ "unofficialSources": ALL_UNOFFICIAL_SOURCES }, tsPromise);
}

export async function getUnofficialSource(request, path) {
const tsPromise = fetchTimestamps();
const arr = path.split('/');
if (arr.length < 3 || !ALL_UNOFFICIAL_SOURCES.includes(arr[2])) {
return errorResponse({ reason: "invalid source" }, tsPromise, 404);
}
else {
const suffix = arr.length > 3 ? `_${arr[3]}` : '';
const data = await Store.get(STORE_KEYS.UNOFFICIAL_SRC_PREFIX + arr[2] + suffix);
return successResponse(JSON.parse(data), tsPromise);
}
return successResponse({ "unofficialSources": ALL_UNOFFICIAL_SOURCES });
}

const COVID19INDIA_ORG = 'covid19india.org';

0 comments on commit b2a5814

Please sign in to comment.