|
10 | 10 |
|
11 | 11 | import * as genai from "oci-generativeaiinference";
|
12 | 12 | import common = require("oci-common");
|
| 13 | +import { Readable } from "stream"; |
| 14 | +import * as readline from "readline"; |
13 | 15 |
|
14 | 16 | (async () => {
|
15 | 17 | const region = "us-chicago-1";
|
@@ -61,27 +63,50 @@ import common = require("oci-common");
|
61 | 63 | .inferenceResponse as genai.models.CohereLlmInferenceResponse).generatedTexts[0].text
|
62 | 64 | );
|
63 | 65 |
|
64 |
| - // Attempt to generate text as SSE stream (throws error) |
| 66 | + // Generate text as stream response |
65 | 67 | try {
|
66 | 68 | inference_request.isStream = true;
|
67 |
| - const responseStream = await client.generateText(req_body); |
68 |
| - |
| 69 | + const [major, minor, patch] = process.versions.node.split(".").map(Number); |
| 70 | + console.log("Version: " + major); |
69 | 71 | let streamData = "";
|
70 |
| - const lines = String(responseStream).split("\n"); |
71 | 72 |
|
72 |
| - lines.forEach(line => { |
73 |
| - if (line.trim() === "") { |
74 |
| - } else { |
75 |
| - if (line.startsWith("data:")) { |
76 |
| - const data = JSON.parse(line.substring(6).trim()); |
77 |
| - streamData += data.text; |
| 73 | + if (major >= 18) { |
| 74 | + // If node version is above 18 |
| 75 | + const readablestream = (await client.generateText(req_body)) as ReadableStream<Uint8Array>; |
| 76 | + const reader = readablestream.pipeThrough(new TextDecoderStream()).getReader(); |
| 77 | + let streamData = ""; |
| 78 | + while (true) { |
| 79 | + const { done, value } = await reader.read(); |
| 80 | + if (done) break; |
| 81 | + if (value) { |
| 82 | + streamData = parseEvent(value); |
| 83 | + process.stdout.write(streamData); |
78 | 84 | }
|
79 | 85 | }
|
80 |
| - }); |
81 |
| - console.log("Stream Response: ", streamData); |
| 86 | + console.log("\n"); |
| 87 | + } else { |
| 88 | + // If node version is below 18 or above 14 |
| 89 | + const readablestream = ((await client.generateText(req_body)) as unknown) as Readable; |
| 90 | + const eventstream = readline.createInterface({ input: readablestream }); |
| 91 | + eventstream.on("line", (chunk: string) => { |
| 92 | + streamData = parseEvent(chunk); |
| 93 | + process.stdout.write(streamData); |
| 94 | + }); |
| 95 | + eventstream.on("close", () => { |
| 96 | + console.log("\n"); |
| 97 | + }); |
| 98 | + } |
82 | 99 | } catch (e) {
|
83 | 100 | console.log(e);
|
84 | 101 | }
|
85 |
| - |
86 |
| - client.close(); |
87 | 102 | })();
|
| 103 | + |
| 104 | +function parseEvent(eventData: string): string { |
| 105 | + if (eventData.startsWith("data:")) { |
| 106 | + const data = JSON.parse(eventData.substring(6).trim()); |
| 107 | + if ("text" in data) { |
| 108 | + return data.text; |
| 109 | + } |
| 110 | + } |
| 111 | + return ""; |
| 112 | +} |
0 commit comments