# チームとプロジェクト名を設定しますweave.init('[YOUR-TEAM]/eval_pipeline_quickstart')model = ExtractFruitsModel( model_name='gpt-3.5-turbo-1106', prompt_template='Extract fields ("fruit": <str>, "color": <str>, "flavor": <str>) from the following text, as json: {sentence}')sentence = "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy."print(asyncio.run(model.predict(sentence)))# Jupyter Notebook を使用している場合は、次を実行します:# await model.predict(sentence)
awaitweave.init('eval_pipeline_quickstart');constsentence = "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.";constresult = awaitmodel({ datasetRow: { sentence } });console.log(result);
Model を定義したら、次はそれを評価するためのデータセットが必要です。Dataset は Weaveオブジェクトとして保存されるサンプルのコレクションです。データセットを Weave に公開するとバージョン管理され、評価 run 間で再利用できるようになります。次のデータセット例では、3 つの入力文のサンプルとそれぞれの正解 (labels) を定義し、スコアリング関数が読み取れる JSON の表形式に整形します。この例では、コード内でサンプルのリストを作成していますが、実行中のアプリケーションから 1 件ずつログすることもできます。
Python
TypeScript
sentences = ["There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.","Pounits are a bright green color and are more savory than sweet.","Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them."]labels = [ {'fruit': 'neoskizzles', 'color': 'purple', 'flavor': 'candy'}, {'fruit': 'pounits', 'color': 'bright green', 'flavor': 'savory'}, {'fruit': 'glowls', 'color': 'pale orange', 'flavor': 'sour and bitter'}]examples = [ {'id': '0', 'sentence': sentences[0], 'target': labels[0]}, {'id': '1', 'sentence': sentences[1], 'target': labels[1]}, {'id': '2', 'sentence': sentences[2], 'target': labels[2]}]
constsentences = [ "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.", "Pounits are a bright green color and are more savory than sweet.", "Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them."];constlabels = [ { fruit: 'neoskizzles', color: 'purple', flavor: 'candy' }, { fruit: 'pounits', color: 'bright green', flavor: 'savory' }, { fruit: 'glowls', color: 'pale orange', flavor: 'sour and bitter' }];constexamples =sentences.map((sentence, i) => ({id:i.toString(),sentence,target:labels[i]}));
import jsonimport asyncioimport openaiimport weavefrom weave.scorers import MultiTaskBinaryClassificationF1# Weave を一度初期化するweave.init('eval_pipeline_quickstart')# 1. モデルを定義するclass ExtractFruitsModel(weave.Model): model_name: str prompt_template: str @weave.op() async def predict(self, sentence: str) -> dict: client = openai.AsyncClient() response = await client.chat.completions.create( model=self.model_name, messages=[{"role": "user", "content": self.prompt_template.format(sentence=sentence)}], ) result = response.choices[0].message.content if result is None: raise ValueError("No response from model") return json.loads(result)# 2. モデルをインスタンス化するmodel = ExtractFruitsModel( model_name='gpt-3.5-turbo-1106', prompt_template='Extract fields ("fruit": <str>, "color": <str>, "flavor": <str>) from the following text, as json: {sentence}')# 3. データセットを作成するsentences = ["There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.","Pounits are a bright green color and are more savory than sweet.","Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them."]labels = [ {'fruit': 'neoskizzles', 'color': 'purple', 'flavor': 'candy'}, {'fruit': 'pounits', 'color': 'bright green', 'flavor': 'savory'}, {'fruit': 'glowls', 'color': 'pale orange', 'flavor': 'sour and bitter'}]examples = [ {'id': '0', 'sentence': sentences[0], 'target': labels[0]}, {'id': '1', 'sentence': sentences[1], 'target': labels[1]}, {'id': '2', 'sentence': sentences[2], 'target': labels[2]}]dataset = weave.Dataset(name='fruits', rows=examples)weave.publish(dataset)# 4. スコアリング関数を定義する@weave.op()def fruit_name_score(target: dict, output: dict) -> dict: return {'correct': target['fruit'] == output['fruit']}# 5. 評価を実行するevaluation = weave.Evaluation( name='fruit_eval', dataset=dataset, scorers=[ MultiTaskBinaryClassificationF1(class_names=["fruit", "color", "flavor"]), fruit_name_score ],)print(asyncio.run(evaluation.evaluate(model)))
import * asweave from 'weave';importOpenAI from 'openai';// Weave を一度初期化するawaitweave.init('eval_pipeline_quickstart');// 1. モデルを定義する// 注意: weave.Model は TypeScript ではまだサポートされていません。// 代わりに、モデルのような関数を weave.op でラップしてください。constopenaiClient = newOpenAI();constmodel =weave.op(async functionmyModel({datasetRow}) { constprompt = `Extract fields ("fruit": <str>, "color": <str>, "flavor": <str>) from the following text, as json: ${datasetRow.sentence}`; constresponse = awaitopenaiClient.chat.completions.create({model: 'gpt-3.5-turbo',messages: [{ role: 'user', content:prompt }],response_format: { type: 'json_object' } }); returnJSON.parse(response.choices[0].message.content);});// 2. データセットを作成するconstsentences = [ "There are many fruits that were found on the recently discovered planet Goocrux. There are neoskizzles that grow there, which are purple and taste like candy.", "Pounits are a bright green color and are more savory than sweet.", "Finally, there are fruits called glowls, which have a very sour and bitter taste which is acidic and caustic, and a pale orange tinge to them."];constlabels = [ { fruit: 'neoskizzles', color: 'purple', flavor: 'candy' }, { fruit: 'pounits', color: 'bright green', flavor: 'savory' }, { fruit: 'glowls', color: 'pale orange', flavor: 'sour and bitter' }];constexamples =sentences.map((sentence, i) => ({id:i.toString(),sentence,target:labels[i]}));constdataset = newweave.Dataset({name: 'fruits',rows:examples});awaitdataset.save();// 3. スコアリング関数を定義するconstfruitNameScorer =weave.op( functionfruitNameScore({target, output}) { return { correct:target.fruit ===output.fruit }; });// 4. 評価を実行するconstevaluation = newweave.Evaluation({name: 'fruit_eval',dataset:dataset,scorers: [fruitNameScorer],});constresults = awaitevaluation.evaluate(model);console.log(results);
