From 8c6a9b8bb6a0273cc0b5915903ca1ff9206c6285 Mon Sep 17 00:00:00 2001 From: KITAITI Makoto Date: Fri, 22 Nov 2024 00:04:29 +0900 Subject: [PATCH] ruby : Follow source tree change (#2580) * Follow whisper.cpp source tree change * Update whispercpp.gemspec * Follow whisper.cpp log level change * Fix paths in GitHub workflow for Ruby bindings * Use GitHub workflow setting for dependency definition * Use ternary operator --- .github/workflows/bindings-ruby.yml | 80 +++++++++++------------------ bindings/ruby/Rakefile | 14 ++--- bindings/ruby/ext/.gitignore | 43 ++++------------ bindings/ruby/ext/cpu.mk | 9 ++++ bindings/ruby/ext/extconf.rb | 39 ++++++++------ bindings/ruby/ext/metal-embed.mk | 29 ++++++----- bindings/ruby/ext/metal.mk | 6 +++ bindings/ruby/extsources.rb | 6 +++ bindings/ruby/extsources.yaml | 31 ----------- bindings/ruby/tests/test_whisper.rb | 2 +- bindings/ruby/whispercpp.gemspec | 10 ++-- 11 files changed, 116 insertions(+), 153 deletions(-) create mode 100644 bindings/ruby/ext/cpu.mk create mode 100644 bindings/ruby/ext/metal.mk create mode 100644 bindings/ruby/extsources.rb delete mode 100644 bindings/ruby/extsources.yaml diff --git a/.github/workflows/bindings-ruby.yml b/.github/workflows/bindings-ruby.yml index d1d3c34..b49e862 100644 --- a/.github/workflows/bindings-ruby.yml +++ b/.github/workflows/bindings-ruby.yml @@ -3,61 +3,41 @@ on: push: paths: - bindings/ruby/** - - src/whisper.cpp - - include/whisper.h - - ggml/src/ggml.c - - ggml/src/ggml-impl.h - - ggml/src/ggml-aarch64.h - - ggml/src/ggml-aarch64.c - - ggml/src/ggml-alloc.c - - ggml/src/ggml-backend-impl.h - - ggml/src/ggml-backend.cpp - - ggml/src/ggml-common.h - - ggml/src/ggml-quants.h - - ggml/src/ggml-quants.c - - ggml/src/ggml-cpu-impl.h - - ggml/src/ggml-metal.m - - ggml/src/ggml-metal.metal - - ggml/src/ggml-blas.cpp - - ggml/include/ggml.h - - ggml/include/ggml-alloc.h - - ggml/include/ggml-backend.h - - ggml/include/ggml-cuda.h - - ggml/include/ggml-kompute.h - - ggml/include/ggml-metal.h - - ggml/include/ggml-sycl.h - - ggml/include/ggml-vulkan.h - - ggml/include/ggml-blas.h + - src/**/*.c + - src/**/*.cpp + - src/**/*.h + - src/**/*.m + - src/**/*.metal + - include/**/*.c + - include/**/*.cpp + - include/**/*.h + - include/**/*.m + - include/**/*.metal + - ggml/**/*.c + - ggml/**/*.cpp + - ggml/**/*.h + - ggml/**/*.m + - ggml/**/*.metal - scripts/get-flags.mk - examples/dr_wav.h pull_request: paths: - bindings/ruby/** - - src/whisper.cpp - - include/whisper.h - - ggml/src/ggml.c - - ggml/src/ggml-impl.h - - ggml/src/ggml-aarch64.h - - ggml/src/ggml-aarch64.c - - ggml/src/ggml-alloc.c - - ggml/src/ggml-backend-impl.h - - ggml/src/ggml-backend.cpp - - ggml/src/ggml-common.h - - ggml/src/ggml-quants.h - - ggml/src/ggml-quants.c - - ggml/src/ggml-cpu-impl.h - - ggml/src/ggml-metal.m - - ggml/src/ggml-metal.metal - - ggml/src/ggml-blas.cpp - - ggml/include/ggml.h - - ggml/include/ggml-alloc.h - - ggml/include/ggml-backend.h - - ggml/include/ggml-cuda.h - - ggml/include/ggml-kompute.h - - ggml/include/ggml-metal.h - - ggml/include/ggml-sycl.h - - ggml/include/ggml-vulkan.h - - ggml/include/ggml-blas.h + - src/**/*.c + - src/**/*.cpp + - src/**/*.h + - src/**/*.m + - src/**/*.metal + - include/**/*.c + - include/**/*.cpp + - include/**/*.h + - include/**/*.m + - include/**/*.metal + - ggml/**/*.c + - ggml/**/*.cpp + - ggml/**/*.h + - ggml/**/*.m + - ggml/**/*.metal - scripts/get-flags.mk - examples/dr_wav.h diff --git a/bindings/ruby/Rakefile b/bindings/ruby/Rakefile index d6fc49c..53eb617 100644 --- a/bindings/ruby/Rakefile +++ b/bindings/ruby/Rakefile @@ -1,20 +1,22 @@ require 'rake/clean' require "bundler/gem_tasks" -require "pathname" -require "yaml" require "rake/testtask" +require_relative "extsources" -extsources = YAML.load_file("extsources.yaml") SOURCES = FileList[] -extsources.each do |src| + +EXTSOURCES.each do |src| basename = src.pathmap("%f") - dest = basename == "LICENSE" ? basename : basename.pathmap("ext/%f") + dest = basename == "LICENSE" ? basename : src.pathmap("%{../..,ext}p") + dir = dest.pathmap("%d") file src - file dest => src do |t| + directory dir + file dest => [src, dir] do |t| cp t.source, t.name end SOURCES.include dest end + CLEAN.include SOURCES CLEAN.include FileList[ "ext/*.o", diff --git a/bindings/ruby/ext/.gitignore b/bindings/ruby/ext/.gitignore index c9f3196..3804ab7 100644 --- a/bindings/ruby/ext/.gitignore +++ b/bindings/ruby/ext/.gitignore @@ -1,35 +1,14 @@ Makefile -ggml.c -ggml.h -ggml-alloc.c -ggml-alloc.h -ggml-aarch64.c -ggml-aarch64.h -ggml-backend.cpp -ggml-backend-impl.h -ggml-backend.c -ggml-backend.h -ggml-common.h -ggml-cpu-impl.h -ggml-metal.m -ggml-metal.metal -ggml-metal-embed.metal -ggml-blas.cpp -ggml-cuda.h -ggml-impl.h -ggml-kompute.h -ggml-metal.h -ggml-opencl.h -ggml-quants.c -ggml-quants.h -ggml-sycl.h -ggml-vulkan.h -ggml-blas.h -get-flags.mk -whisper.cpp -whisper.h -dr_wav.h -depend -whisper.bundle whisper.so +whisper.bundle whisper.dll +depend +scripts/get-flags.mk +*.o +*.c +*.cpp +*.h +*.m +*.metal +!ruby_whisper.cpp +!ruby_whisper.h diff --git a/bindings/ruby/ext/cpu.mk b/bindings/ruby/ext/cpu.mk new file mode 100644 index 0000000..a6aa237 --- /dev/null +++ b/bindings/ruby/ext/cpu.mk @@ -0,0 +1,9 @@ +ggml/src/ggml-cpu/ggml-cpu-cpp.o: \ + ggml/src/ggml-cpu/ggml-cpu.cpp \ + ggml/include/ggml-backend.h \ + ggml/include/ggml.h \ + ggml/include/ggml-alloc.h \ + ggml/src/ggml-backend-impl.h \ + ggml/include/ggml-cpu.h \ + ggml/src/ggml-impl.h + $(CXX) $(CXXFLAGS) -c $< -o $@ diff --git a/bindings/ruby/ext/extconf.rb b/bindings/ruby/ext/extconf.rb index dd4db09..6d76a7c 100644 --- a/bindings/ruby/ext/extconf.rb +++ b/bindings/ruby/ext/extconf.rb @@ -35,7 +35,7 @@ if $GGML_METAL $GGML_METAL_EMBED_LIBRARY = true end -$MK_CPPFLAGS = '' +$MK_CPPFLAGS = '-Iggml/include -Iggml/src -Iinclude -Isrc -Iexamples' $MK_CFLAGS = '-std=c11 -fPIC' $MK_CXXFLAGS = '-std=c++11 -fPIC' $MK_NVCCFLAGS = '-std=c++11' @@ -123,11 +123,11 @@ end unless ENV['GGML_NO_ACCELERATE'] if $UNAME_S == 'Darwin' - $MK_CPPFLAGS << ' -DGGML_USE_ACCELERATE -DGGML_USE_BLAS' + $MK_CPPFLAGS << ' -DGGML_USE_ACCELERATE -DGGML_USE_BLAS -DGGML_BLAS_USE_ACCELERATE' $MK_CPPFLAGS << ' -DACCELERATE_NEW_LAPACK' $MK_CPPFLAGS << ' -DACCELERATE_LAPACK_ILP64' $MK_LDFLAGS << ' -framework Accelerate' - $OBJ_GGML << 'ggml-blas.o' + $OBJ_GGML << 'ggml/src/ggml-blas/ggml-blas.o' end end @@ -135,20 +135,20 @@ if ENV['GGML_OPENBLAS'] $MK_CPPFLAGS << " -DGGML_USE_BLAS #{`pkg-config --cflags-only-I openblas`.chomp}" $MK_CFLAGS << " #{`pkg-config --cflags-only-other openblas)`.chomp}" $MK_LDFLAGS << " #{`pkg-config --libs openblas`}" - $OBJ_GGML << 'ggml-blas.o' + $OBJ_GGML << 'ggml/src/ggml-blas/ggml-blas.o' end if ENV['GGML_OPENBLAS64'] $MK_CPPFLAGS << " -DGGML_USE_BLAS #{`pkg-config --cflags-only-I openblas64`.chomp}" $MK_CFLAGS << " #{`pkg-config --cflags-only-other openblas64)`.chomp}" $MK_LDFLAGS << " #{`pkg-config --libs openblas64`}" - $OBJ_GGML << 'ggml-blas.o' + $OBJ_GGML << 'ggml/src/ggml-blas/ggml-blas.o' end if $GGML_METAL $MK_CPPFLAGS << ' -DGGML_USE_METAL' $MK_LDFLAGS << ' -framework Foundation -framework Metal -framework MetalKit' - $OBJ_GGML << 'ggml-metal.o' + $OBJ_GGML << 'ggml/src/ggml-metal/ggml-metal.o' if ENV['GGML_METAL_NDEBUG'] $MK_CPPFLAGS << ' -DGGML_METAL_NDEBUG' @@ -156,20 +156,26 @@ if $GGML_METAL if $GGML_METAL_EMBED_LIBRARY $MK_CPPFLAGS << ' -DGGML_METAL_EMBED_LIBRARY' - $OBJ_GGML << 'ggml-metal-embed.o' + $OBJ_GGML << 'ggml/src/ggml-metal/ggml-metal-embed.o' end end $OBJ_GGML << - 'ggml.o' << - 'ggml-cpu.o' << - 'ggml-alloc.o' << - 'ggml-backend.o' << - 'ggml-quants.o' << - 'ggml-aarch64.o' + 'ggml/src/ggml.o' << + 'ggml/src/ggml-aarch64.o' << + 'ggml/src/ggml-alloc.o' << + 'ggml/src/ggml-backend.o' << + 'ggml/src/ggml-backend-reg.o' << + 'ggml/src/ggml-opt.o' << + 'ggml/src/ggml-quants.o' << + 'ggml/src/ggml-threading.o' << + 'ggml/src/ggml-cpu/ggml-cpu.o' << + 'ggml/src/ggml-cpu/ggml-cpu-cpp.o' << + 'ggml/src/ggml-cpu/ggml-cpu-aarch64.o' << + 'ggml/src/ggml-cpu/ggml-cpu-quants.o' $OBJ_WHISPER << - 'whisper.o' + 'src/whisper.o' $objs = $OBJ_GGML + $OBJ_WHISPER + $OBJ_COMMON + $OBJ_SDL $objs << "ruby_whisper.o" @@ -184,9 +190,12 @@ $LDFLAGS = "#{$MK_LDFLAGS} #{$LDFLAGS}" create_makefile('whisper') File.open 'Makefile', 'a' do |file| - file.puts 'include get-flags.mk' + file.puts 'include scripts/get-flags.mk' + file.puts 'include cpu.mk' if $GGML_METAL + file.puts 'include metal.mk' + if $GGML_METAL_EMBED_LIBRARY file.puts 'include metal-embed.mk' end diff --git a/bindings/ruby/ext/metal-embed.mk b/bindings/ruby/ext/metal-embed.mk index 478b8fd..cad86f8 100644 --- a/bindings/ruby/ext/metal-embed.mk +++ b/bindings/ruby/ext/metal-embed.mk @@ -1,14 +1,17 @@ -ggml-metal-embed.o: \ - ggml-metal.metal \ - ggml-common.h +ggml/src/ggml-metal/ggml-metal-embed.o: \ + ggml/src/ggml-metal/ggml-metal.metal \ + ggml/src/ggml-metal/ggml-metal-impl.h \ + ggml/src/ggml-common.h @echo "Embedding Metal library" - @sed -e '/#include "ggml-common.h"/r ggml-common.h' -e '/#include "ggml-common.h"/d' < ggml-metal.metal > ggml-metal-embed.metal - $(eval TEMP_ASSEMBLY=$(shell mktemp)) - @echo ".section __DATA, __ggml_metallib" > $(TEMP_ASSEMBLY) - @echo ".globl _ggml_metallib_start" >> $(TEMP_ASSEMBLY) - @echo "_ggml_metallib_start:" >> $(TEMP_ASSEMBLY) - @echo ".incbin \"ggml-metal-embed.metal\"" >> $(TEMP_ASSEMBLY) - @echo ".globl _ggml_metallib_end" >> $(TEMP_ASSEMBLY) - @echo "_ggml_metallib_end:" >> $(TEMP_ASSEMBLY) - @$(AS) $(TEMP_ASSEMBLY) -o $@ - @rm -f ${TEMP_ASSEMBLY} + @sed -e '/__embed_ggml-common.h__/r ggml/src/ggml-common.h' -e '/__embed_ggml-common.h__/d' < ggml/src/ggml-metal/ggml-metal.metal > ggml/src/ggml-metal/ggml-metal-embed.metal.tmp + @sed -e '/#include "ggml-metal-impl.h"/r ggml/src/ggml-metal/ggml-metal-impl.h' -e '/#include "ggml-metal-impl.h"/d' < ggml/src/ggml-metal/ggml-metal-embed.metal.tmp > ggml/src/ggml-metal/ggml-metal-embed.metal + $(eval TEMP_ASSEMBLY=$(shell mktemp -d)) + @echo ".section __DATA, __ggml_metallib" > $(TEMP_ASSEMBLY)/ggml-metal-embed.s + @echo ".globl _ggml_metallib_start" >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s + @echo "_ggml_metallib_start:" >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s + @echo ".incbin \"ggml/src/ggml-metal/ggml-metal-embed.metal\"" >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s + @echo ".globl _ggml_metallib_end" >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s + @echo "_ggml_metallib_end:" >> $(TEMP_ASSEMBLY)/ggml-metal-embed.s + $(CC) $(CFLAGS) -c $(TEMP_ASSEMBLY)/ggml-metal-embed.s -o $@ + @rm -f ${TEMP_ASSEMBLY}/ggml-metal-embed.s + @rmdir ${TEMP_ASSEMBLY} diff --git a/bindings/ruby/ext/metal.mk b/bindings/ruby/ext/metal.mk new file mode 100644 index 0000000..2b53cde --- /dev/null +++ b/bindings/ruby/ext/metal.mk @@ -0,0 +1,6 @@ +ggml/src/ggml-metal/ggml-metal.o: \ + ggml/src/ggml-metal/ggml-metal.m \ + ggml/src/ggml-metal/ggml-metal-impl.h \ + ggml/include/ggml-metal.h \ + ggml/include/ggml.h + $(CC) $(CFLAGS) -c $< -o $@ diff --git a/bindings/ruby/extsources.rb b/bindings/ruby/extsources.rb new file mode 100644 index 0000000..1dc900d --- /dev/null +++ b/bindings/ruby/extsources.rb @@ -0,0 +1,6 @@ +require "yaml" + +sources = `git ls-files -z ../..`.split("\x0") +paths = YAML.load_file("../../.github/workflows/bindings-ruby.yml")[true]["push"]["paths"] +paths.delete "bindings/ruby/**" +EXTSOURCES = (Dir.glob(paths, base: "../..").collect {|path| "../../#{path}"} << "../../LICENSE") & sources diff --git a/bindings/ruby/extsources.yaml b/bindings/ruby/extsources.yaml deleted file mode 100644 index 10c2a56..0000000 --- a/bindings/ruby/extsources.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -- ../../src/whisper.cpp -- ../../include/whisper.h -- ../../ggml/src/ggml.c -- ../../ggml/src/ggml-cpu.c -- ../../ggml/src/ggml-impl.h -- ../../ggml/src/ggml-aarch64.h -- ../../ggml/src/ggml-aarch64.c -- ../../ggml/src/ggml-alloc.c -- ../../ggml/src/ggml-backend-impl.h -- ../../ggml/src/ggml-backend.cpp -- ../../ggml/src/ggml-common.h -- ../../ggml/src/ggml-quants.h -- ../../ggml/src/ggml-quants.c -- ../../ggml/src/ggml-cpu-impl.h -- ../../ggml/src/ggml-metal.m -- ../../ggml/src/ggml-metal.metal -- ../../ggml/src/ggml-blas.cpp -- ../../ggml/include/ggml.h -- ../../ggml/include/ggml-alloc.h -- ../../ggml/include/ggml-backend.h -- ../../ggml/include/ggml-cpu.h -- ../../ggml/include/ggml-cuda.h -- ../../ggml/include/ggml-kompute.h -- ../../ggml/include/ggml-metal.h -- ../../ggml/include/ggml-sycl.h -- ../../ggml/include/ggml-vulkan.h -- ../../ggml/include/ggml-blas.h -- ../../scripts/get-flags.mk -- ../../examples/dr_wav.h -- ../../LICENSE diff --git a/bindings/ruby/tests/test_whisper.rb b/bindings/ruby/tests/test_whisper.rb index e37e24c..b488eee 100644 --- a/bindings/ruby/tests/test_whisper.rb +++ b/bindings/ruby/tests/test_whisper.rb @@ -107,7 +107,7 @@ class TestWhisper < TestBase assert logs.length > 30 logs.each do |log| - assert_equal Whisper::LOG_LEVEL_INFO, log[0] + assert_include [Whisper::LOG_LEVEL_DEBUG, Whisper::LOG_LEVEL_INFO, Whisper::LOG_LEVEL_WARN], log[0] assert_same user_data, log[2] end end diff --git a/bindings/ruby/whispercpp.gemspec b/bindings/ruby/whispercpp.gemspec index 251d03f..2fd9cad 100644 --- a/bindings/ruby/whispercpp.gemspec +++ b/bindings/ruby/whispercpp.gemspec @@ -1,4 +1,4 @@ -require "yaml" +require_relative "extsources" Gem::Specification.new do |s| s.name = "whispercpp" @@ -10,24 +10,24 @@ Gem::Specification.new do |s| s.extra_rdoc_files = ['LICENSE', 'README.md'] s.files = `git ls-files . -z`.split("\x0") + - YAML.load_file("extsources.yaml").collect {|file| + EXTSOURCES.collect {|file| basename = File.basename(file) if s.extra_rdoc_files.include?(basename) basename else - File.join("ext", basename) + file.sub("../..", "ext") end } s.summary = %q{Ruby whisper.cpp bindings} - s.test_files = ["tests/test_whisper.rb"] + s.test_files = s.files.select {|file| file.start_with? "tests/"} s.extensions << 'ext/extconf.rb' #### Documentation and testing. s.homepage = 'https://github.com/ggerganov/whisper.cpp' - s.rdoc_options = ['--main', '../../README.md'] + s.rdoc_options = ['--main', 'README.md'] s.platform = Gem::Platform::RUBY