प्रदर्शन को बढ़ावा दें:लैंगचेन, रेडिस और क्यूस्टैश के साथ ऑन-डिमांड एलएलएम कैशिंग

आवश्यकताएँ

• एक अपस्टैश रेडिस डेटाबेस
• QStash पर्यावरण चर
• एक OpenAI API कुंजी

आरंभ करना

प्रोजेक्ट बनाना

mkdir langchain-qstash
cd langchain-qstash

bun create hono@latest

निर्भरताएं स्थापित करना

bun install @upstash/qstash @upstash/ratelimit @upstash/redis langchain openai

प्रोजेक्ट को कॉन्फ़िगर करना

node_modules
dist
.wrangler
.dev.vars
wrangler.toml

[vars]
QSTASH_CURRENT_SIGNING_KEY="sig_********"
QSTASH_NEXT_SIGNING_KEY="sig_********"
UPSTASH_REDIS_REST_URL="https://********.upstash.io"
UPSTASH_REDIS_REST_TOKEN="********"
OPENAI_API_KEY="sk-********"

type Bindings = {
 QSTASH_CURRENT_SIGNING_KEY: string;
 QSTASH_NEXT_SIGNING_KEY: string;
 UPSTASH_REDIS_REST_URL: string;
 UPSTASH_REDIS_REST_TOKEN: string;
 OPENAI_API_KEY: string;
};
 
const app = new Hono<{ Bindings: Bindings }>();

विकास

bun run dev

"dev": "wrangler dev src/index.ts --remote",

bun run deploy

मिडिलवेयर बनाना

import { Hono } from "hono";
import { logger } from "hono/logger";
 
// snip
 
const app = new Hono<{ Bindings: Bindings }>();
app.use("*", logger());
 
// snip

QStash को क्लाउडफ्लेयर वर्कर्स से जोड़ना

import { Receiver } from "@upstash/qstash";
import { type MiddlewareHandler } from "hono";
 
declare global {
 interface Response {
 locals: {
 query: string;
 };
 }
}
 
export const verify: MiddlewareHandler = async (ctx, next) => {
 const receiver = new Receiver({
 currentSigningKey: ctx.env.QSTASH_CURRENT_SIGNING_KEY,
 nextSigningKey: ctx.env.QSTASH_NEXT_SIGNING_KEY,
 });
};

// snip
 
const body = await ctx.req.text();
ctx.res.locals = {
 query: JSON.parse(body).query,
};
 
const isValid = await receiver
 .verify({
 signature: ctx.req.headers.get("Upstash-Signature")!,
 body,
 })
 .catch((err) => {
 console.error(err);
 return false;
 });
 
if (!isValid) {
 return new Response("Invalid signature", { status: 401 });
}
 
await next();

दर-सीमित जोड़ना

import { Ratelimit } from "@upstash/ratelimit";
import { Redis } from "@upstash/redis/cloudflare";
import { type MiddlewareHandler } from "hono";
 
export const ratelimit: MiddlewareHandler = async (ctx, next) => {
 const redis = new Redis({
 url: ctx.env.UPSTASH_REDIS_REST_URL,
 token: ctx.env.UPSTASH_REDIS_REST_TOKEN,
 });
 
 const ratelimit = new Ratelimit({
 redis,
 limiter: Ratelimit.slidingWindow(10, "10 s"),
 analytics: true,
 });
 
 await next();
};

const cache = new Map(); // outside of the middleware handler
 
// snip
 
const ratelimit = new Ratelimit({
 ephemeralCache: cache,
 // snip
});

// snip
 
const identifier = "openai";
const { success } = await ratelimit.limit(identifier);
 
if (!success) {
 return new Response("Too many requests", { status: 429 });
}
 
await next();
// snip

const { pending, success } = await ratelimit.limit(identifier);
 
ctx.event.waitUntil(pending);

QStash से संदेश प्राप्त करना

// snip
import { ratelimit } from "./middleware/ratelimit";
import { verify } from "./middleware/verify";
 
// snip
 
app.post("/api/announce", ratelimit, verify, async (ctx) => {});
 
// snip

// snip
 
import { Redis } from "@upstash/redis/cloudflare";
import { UpstashRedisCache } from "langchain/cache/upstash_redis";
import { OpenAI } from "langchain/llms/openai";
 
// snip
 
app.post("/api/announce", ratelimit, verify, async (ctx) => {
 const redis = new Redis({
 url: ctx.env.UPSTASH_REDIS_REST_URL,
 token: ctx.env.UPSTASH_REDIS_REST_TOKEN,
 });
 
 const cache = new UpstashRedisCache({ client: redis });
 const model = new OpenAI({
 cache,
 openAIApiKey: ctx.env.OPENAI_API_KEY,
 });
 
 const query = ctx.res.locals.query;
 const result = await model
 .call(query)
 .then((result) => {
 console.log(result);
 return result;
 })
 .catch((err) => console.error(err));
 
 return new Response(result ?? "", { status: 200 });
});
 
// snip

bun run deploy

curl -XPOST \
 "https://qstash.upstash.io/v2/publish/https://<YOUR_API_URL>.workers.dev/api/announce" \
 -H "Authorization: Bearer <YOUR_QSTASH_TOKEN>" \
 -H "Content-Type: application/json" \
 -d "{ \"query\": \"What's the derivative of e^x?\" }"

{
 "query": "What's the derivative of e^x?"
}

निष्कर्ष