OpenAI APIを使って画像に写っているものを説明してもらう 📷

前回は、Node.jsからOpen AIのAPIをさささっと叩いてみることを目標に、適当な日本語を返してもらいましたが、今回は画像に写っているものを説明してもらおうと思います。

blog.kimizuka.org

import OpenAI from 'openai';

const openai = new OpenAI({
  apiKey: process.env.API_KEY
});

const chatCompletion = await openai.chat.completions.create({
  model: 'gpt-4-vision-preview',
  messages: [{
    role: 'user',
    content: [{
      type: 'text',
      text: 'この画像には何が写っているか教えてください'
    },{
      type: 'image_url',
      image_url: 画像のURL
    }]
  }]
});

res.status(200).json({ state: 'ok', data: chatCompletion.choices[0] });

gpt-4-vision-previewを使って、こんな感じで画像の説明を取得しようと思いますが、画像のURLの部分には文字通り画像のURLが入るのですが、ハードコーディングしてしまうと差し替えに手間がかかってしまいます。
なので、簡単に差し替えられるようにNext.jsを使ってブラウザから画像のbase64を渡せるようにしてみます。

ソースコード

package.json

{
  "name": "openai-api-develop",
  "version": "1.0.0",
  "license": "MIT",
  "scripts": {
    "dev": "next dev",
    "build": "next build",
    "start": "next start",
    "lint": "next lint"
  },
  "dependencies": {
    "axios": "^1.6.7",
    "next": "14.1.0",
    "openai": "^4.26.0",
    "react": "^18",
    "react-dom": "^18",
    "styled-components": "^6.1.8"
  },
  "devDependencies": {
    "@types/node": "^20",
    "@types/react": "^18",
    "@types/react-dom": "^18",
    "@types/styled-components": "^5.1.34",
    "babel-plugin-styled-components": "^2.1.4",
    "eslint": "^8",
    "eslint-config-next": "14.1.0",
    "sass": "^1.70.0",
    "typescript": "^5"
  }
}

src/pages/api/openai.ts

import OpenAI from 'openai';
import type { NextApiRequest, NextApiResponse } from 'next';

type Data = {
  state: string;
  data?: any;
}

export default async function handler(
  req: NextApiRequest,
  res: NextApiResponse<Data>
) {
  if (req.method === 'POST') {
    const openai = new OpenAI({
      apiKey: process.env.OPENAI_API_KEY
    });

    const chatCompletion = await openai.chat.completions.create({
      model: 'gpt-4-vision-preview',
      messages: [{
        role: 'user',
        content: [{
          type: 'text',
          text: 'この画像には何が写っているか教えてください'
        },{
          type: 'image_url',
          image_url: req.body.base64
        }]
      }],
      max_tokens: 300
    });

    res.status(200).json({ state: 'ok', data: chatCompletion.choices[0] });
  }
}

src/pages/index.tsx

import Head from 'next/head';
import { IndexPageTemplate } from '@/components/templates/IndexPageTemplate';

export default function IndexPage() {
  return (
    <>
      <Head>
        <title>openai-api-develop</title>
        <meta name="description" content="Generated by create next app" />
        <meta name="viewport" content="width=device-width, initial-scale=1" />
        <link rel="icon" href="/favicon.ico" />
      </Head>
      <≈ />
    </>
  )
}

src/components/templates/IndexPageTemplate

import axios from 'axios';
import styled from 'styled-components';
import { FormEvent, useCallback, useRef, useState } from 'react';

export function IndexPageTemplate() {
  const canvasRef = useRef<HTMLCanvasElement>(null);
  const [ base64, setBase64 ] = useState<string>('');
  const [ text, setText ] = useState<string>('');
  const [ isLoading, setIsLoading ] = useState<boolean>(false);

  const handleClickBtnSubmit = useCallback(async () => {
    if (base64) {
      setIsLoading(true);
      const { data } = await axios.post('/api/openai', { base64 });

      setText(data.data.message.content);
      setIsLoading(false);
    }
  }, [base64]);

  function handleChangeInput(evt: FormEvent<HTMLInputElement>) {
    const file = evt.currentTarget.files?.[0];

    if (file) {
      const reader = new FileReader();

      reader.onload = () => {
        const image = new Image();

        image.onload = () => {
          const canvas = canvasRef.current;
          const ctx = canvas?.getContext('2d');

          if (canvas && ctx) {
            canvas.width = Math.min(640, image.width);
            canvas.height = image.height * (canvas.width / image.width);

            ctx.drawImage(image, 0, 0, canvas.width, canvas.height);
            setBase64(canvas.toDataURL('image/jpeg'));
          }
        }

        image.src = reader.result as string;
      }

      reader.readAsDataURL(file);
    }
  }

  return (
    <Wrapper data-is-loading={ isLoading }>
      <input onChange={ handleChangeInput } type="file" accept="image/*" capture="environment" />
      <canvas ref={ canvasRef } />
      <button onClick={ handleClickBtnSubmit }>submit</button>
      <p>{ text }</p>
      <div className="overlay">
        <svg width="25" height="24" viewBox="0 0 25 24" fill="none" xmlns="http://www.w3.org/2000/svg">
          <circle opacity="0.5" cx="12.5" cy="12" r="10" stroke="white" strokeWidth="4"/>
          <path d="M22.5 12C22.5 6.47715 18.0228 2 12.5 2" stroke="white" strokeWidth="4" strokeLinecap="round">
            <animateTransform
              attributeName="transform"
              type="rotate"
              repeatCount="indefinite"
              dur=".8s"
              from="0 12 12"
              to="360 12 12"
            />
          </path>
        </svg>
      </div>
    </Wrapper>
  );
}

const Wrapper = styled.div`
  padding: 16px;

  canvas {
    display: block;
    max-width: 100%;
    height: auto;
  }

  .overlay {
    position: fixed;
    top: 0; bottom: 0;
    left: 0; right: 0;
    background: rgba(0, 0, 0, 0.4);
    opacity: 0;
    pointer-events: none;
  }

  svg {
    position: fixed;
    top: 50%;
    left: 50%;
    transform: translate(-50%, -50%);
  }

  &[data-is-loading='true'] {
    pointer-events: none;

    .overlay {
      opacity: 1;
    }
  }
`;