RAG アプリケーションを評価する

require('dotenv').config();
import { OpenAI } from 'openai';
import * as weave from 'weave';

interface Article {
    text: string;
    embedding?: number[];
}

const articles: Article[] = [
    { 
        text: `Novo Nordisk and Eli Lilly rival soars 32 percent after promising weight loss drug results Shares of Denmarks Zealand Pharma shot 32 percent higher in morning trade, after results showed success in its liver disease treatment survodutide, which is also on trial as a drug to treat obesity. The trial tells us that the 6mg dose is safe, which is the top dose used in the ongoing [Phase 3] obesity trial too, one analyst said in a note. The results come amid feverish investor interest in drugs that can be used for weight loss.`
    },
    { 
        text: `Berkshire shares jump after big profit gain as Buffetts conglomerate nears $1 trillion valuation Berkshire Hathaway shares rose on Monday after Warren Buffetts conglomerate posted strong earnings for the fourth quarter over the weekend. Berkshires Class A and B shares jumped more than 1.5%, each. Class A shares are higher by more than 17% this year, while Class B has gained more than 18%. Berkshire was last valued at $930.1 billion, up from $905.5 billion where it closed on Friday, according to FactSet. Berkshire on Saturday posted fourth-quarter operating earnings of $8.481 billion, about 28 percent higher than the $6.625 billion from the year-ago period, driven by big gains in its insurance business. Operating earnings refers to profits from businesses across insurance, railroads and utilities. Meanwhile, Berkshires cash levels also swelled to record levels. The conglomerate held $167.6 billion in cash in the fourth quarter, surpassing the $157.2 billion record the conglomerate held in the prior quarter.`
    },
    { 
        text: `Highmark Health says its combining tech from Google and Epic to give doctors easier access to information Highmark Health announced it is integrating technology from Google Cloud and the health-care software company Epic Systems. The integration aims to make it easier for both payers and providers to access key information they need, even if its stored across multiple points and formats, the company said. Highmark is the parent company of a health plan with 7 million members, a provider network of 14 hospitals and other entities`
    }
];

function cosineSimilarity(a: number[], b: number[]): number {
    const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0);
    const magnitudeA = Math.sqrt(a.reduce((sum, val) => sum + val * val, 0));
    const magnitudeB = Math.sqrt(b.reduce((sum, val) => sum + val * val, 0));
    return dotProduct / (magnitudeA * magnitudeB);
}

const docsToEmbeddings = weave.op(async function(docs: Article[]): Promise<Article[]> {
    const openai = new OpenAI();
    const enrichedDocs = await Promise.all(docs.map(async (doc) => {
        const response = await openai.embeddings.create({
            input: doc.text,
            model: "text-embedding-3-small"
        });
        return {
            ...doc,
            embedding: response.data[0].embedding
        };
    }));
    return enrichedDocs;
});

class RAGModel {
    private openai: OpenAI;
    private systemMessage: string;
    private modelName: string;
    private articleEmbeddings: Article[];

    constructor(config: {
        systemMessage: string;
        modelName?: string;
        articleEmbeddings: Article[];
    }) {
        this.openai = new OpenAI();
        this.systemMessage = config.systemMessage;
        this.modelName = config.modelName || "gpt-3.5-turbo-1106";
        this.articleEmbeddings = config.articleEmbeddings;
        this.predict = weave.op(this, this.predict);
    }

    async predict(question: string): Promise<{
        answer: string;
        context: string;
    }> {
        const context = await this.getMostRelevantDocument(question);
        
        const response = await this.openai.chat.completions.create({
            model: this.modelName,
            messages: [
                { role: "system", content: this.systemMessage },
                { role: "user", content: `Use the following information to answer the subsequent question. If the answer cannot be found, write "I don't know."
                    Context:
                    """
                    ${context}
                    """
                    Question: ${question}` }
            ],
            temperature: 0
        });

        return {
            answer: response.choices[0].message.content || "",
            context
        };
    }
}

const contextPrecisionScore = weave.op(async function(args: {
    datasetRow: QuestionRow;
    modelOutput: { answer: string; context: string; }
}): Promise<ScorerResult> {
    const openai = new OpenAI();
    
    const prompt = `Given question, answer and context verify if the context was useful...`;

    const response = await openai.chat.completions.create({
        model: "gpt-4-turbo-preview",
        messages: [{ role: "user", content: prompt }],
        response_format: { type: "json_object" }
    });

    const result = JSON.parse(response.choices[0].message.content || "{}");
    return {
        verdict: parseInt(result.verdict) === 1
    };
});

const evaluation = new weave.Evaluation({
    dataset: createQuestionDataset(),
    scorers: [contextPrecisionScore]
});

await evaluation.evaluate({
    model: weave.op((args: { datasetRow: QuestionRow }) => 
        model.predict(args.datasetRow.question)
    )
});

require('dotenv').config();
import { OpenAI } from 'openai';
import * as weave from 'weave';

interface Article {
    text: string;
    embedding?: number[];
}

const articles: Article[] = [
    { 
        text: `ノボ・ノルディスクとイーライ・リリーのライバル企業、有望な減量薬の結果を受けて株価32%急騰 デンマークのZealand Pharmaの株価は、肝疾患治療薬survodutideの試験で成功が示されたことを受け、午前の取引で32%上昇した。同薬は肥満治療薬としても臨床試験中である。「この試験により、6mgの用量が安全であることが確認された。これは現在進行中の[フェーズ3]肥満試験でも使用されている最高用量だ」とあるアナリストはレポートで述べた。この結果は、減量に使用できる薬への投資家の熱狂的な関心が高まる中で発表された。`
    },
    { 
        text: `バークシャーの株価、大幅増益を受けて急騰——バフェットのコングロマリットが時価総額1兆ドルに迫る ウォーレン・バフェット率いるBerkshire Hathawayが週末に第4四半期の好決算を発表したことを受け、同社株は月曜日に上昇した。クラスAおよびクラスB株はそれぞれ1.5%超上昇した。クラスA株は今年に入り17%超、クラスB株は18%超の上昇となっている。FactSetによると、バークシャーの時価総額は直近で9,301億ドルとなり、金曜日の終値時点の9,055億ドルから上昇した。バークシャーは土曜日、第4四半期の営業利益が84億8,100万ドルと発表した。これは前年同期の66億2,500万ドルから約28%増加しており、保険事業の大幅な利益増が牽引した。営業利益とは、保険、鉄道、公益事業などの各事業からの利益を指す。一方、バークシャーの現金保有額も過去最高水準に膨らんだ。同コングロマリットの第4四半期末の現金保有額は1,676億ドルとなり、前四半期に記録した1,572億ドルを上回った。`
    },
    { 
        text: `Highmark Health、GoogleとEpicの技術を統合して医師の情報アクセスを容易に Highmark Healthは、Google Cloudおよび医療ソフトウェア企業Epic Systemsの技術を統合すると発表した。同社によると、このインテグレーションは、情報が複数の場所や形式にまたがって保存されている場合でも、支払者とプロバイダーの双方が必要な重要情報にアクセスしやすくすることを目的としている。Highmarkは700万人の会員を持つ医療保険プランと、14の病院およびその他の関連組織からなるプロバイダーネットワークの親会社である。`
    }
];

function cosineSimilarity(a: number[], b: number[]): number {
    const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0);
    const magnitudeA = Math.sqrt(a.reduce((sum, val) => sum + val * val, 0));
    const magnitudeB = Math.sqrt(b.reduce((sum, val) => sum + val * val, 0));
    return dotProduct / (magnitudeA * magnitudeB);
}

const docsToEmbeddings = weave.op(async function(docs: Article[]): Promise<Article[]> {
    const openai = new OpenAI();
    const enrichedDocs = await Promise.all(docs.map(async (doc) => {
        const response = await openai.embeddings.create({
            input: doc.text,
            model: "text-embedding-3-small"
        });
        return {
            ...doc,
            embedding: response.data[0].embedding
        };
    }));
    return enrichedDocs;
});

class RAGModel {
    private openai: OpenAI;
    private systemMessage: string;
    private modelName: string;
    private articleEmbeddings: Article[];

    constructor(config: {
        systemMessage: string;
        modelName?: string;
        articleEmbeddings: Article[];
    }) {
        this.openai = new OpenAI();
        this.systemMessage = config.systemMessage;
        this.modelName = config.modelName || "gpt-3.5-turbo-1106";
        this.articleEmbeddings = config.articleEmbeddings;
        this.predict = weave.op(this, this.predict);
    }

    private async getMostRelevantDocument(query: string): Promise<string> {
        const queryEmbedding = await this.openai.embeddings.create({
            input: query,
            model: "text-embedding-3-small"
        });

        const similarities = this.articleEmbeddings.map(doc => {
            if (!doc.embedding) return 0;
            return cosineSimilarity(queryEmbedding.data[0].embedding, doc.embedding);
        });

        const mostRelevantIndex = similarities.indexOf(Math.max(...similarities));
        return this.articleEmbeddings[mostRelevantIndex].text;
    }

    async predict(question: string): Promise<{
        answer: string;
        context: string;
    }> {
        const context = await this.getMostRelevantDocument(question);
        
        const response = await this.openai.chat.completions.create({
            model: this.modelName,
            messages: [
                { role: "system", content: this.systemMessage },
                { 
                    role: "user", 
                    content: `以下の情報を参考に、後続の質問に回答してください。回答が見つからない場合は「わかりません」と記述してください。
                    コンテキスト:
                    """
                    ${context}
                    """
                    質問: ${question}`
                }
            ],
            temperature: 0
        });

        return {
            answer: response.choices[0].message.content || "",
            context
        };
    }
}

interface ScorerResult {
    verdict: boolean;
}

interface QuestionRow {
    question: string;
}

function createQuestionDataset(): weave.Dataset<QuestionRow> {
    return new weave.Dataset<QuestionRow>({
        id: 'rag-questions',
        rows: [
            { question: "Zealand Pharmaの肥満試験について、どのような重要な結果が報告されましたか？" },
            { question: "Berkshire Hathawayの第4四半期における現金保有額はどれだけ増加しましたか？" },
            { question: "Highmark HealthによるGoogle CloudとEpic Systemsの技術統合の目的は何ですか？" }
        ]
    });
}

const contextPrecisionScore = weave.op(async function(args: {
    datasetRow: QuestionRow;
    modelOutput: { answer: string; context: string; }
}): Promise<ScorerResult> {
    const openai = new OpenAI();
    
    const prompt = `質問、回答、およびコンテキストをもとに、そのコンテキストが回答の導出に役立ったかどうかを検証してください。役立った場合は "1"、そうでない場合は "0" をJSON形式で出力してください。
    有効なJSON形式のみで出力してください。

    question: ${args.datasetRow.question}
    context: ${args.modelOutput.context}
    answer: ${args.modelOutput.answer}
    verdict: `;

    const response = await openai.chat.completions.create({
        model: "gpt-4-turbo-preview",
        messages: [{ role: "user", content: prompt }],
        response_format: { type: "json_object" }
    });

    const result = JSON.parse(response.choices[0].message.content || "{}");
    return {
        verdict: parseInt(result.verdict) === 1
    };
});

async function main() {
    # チームとプロジェクト名を設定する
    await weave.init('<team-name>/rag-quickstart');
    
    const articleEmbeddings = await docsToEmbeddings(articles);
    
    const model = new RAGModel({
        systemMessage: "あなたは金融の専門家であり、金融、金融サービス、および金融市場に関する質問に回答します。提供された情報に基づいて回答する際は、必ずソースを引用してください。",
        articleEmbeddings
    });

    const evaluation = new weave.Evaluation({
        dataset: createQuestionDataset(),
        scorers: [contextPrecisionScore]
    });

    const results = await evaluation.evaluate({
        model: weave.op((args: { datasetRow: QuestionRow }) => 
            model.predict(args.datasetRow.question)
        )
    });
    
    console.log('評価結果:', results);
}

if (require.main === module) {
    main().catch(console.error);
}