checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 5f1407ac3ed8008b5fe9016cc6b5497d7e4670c6a3053e82717bbca7ddcafe31
4
+ data.tar.gz: db73972c63544d2ee87ae2df508f38b813cddce5e6e6a8bf8c83012a5fa96a3c
5
+ SHA512:
6
+ metadata.gz: 96793bf87478c3ae07b1d6338813a2fd1f49e89d9b016c02f94bbfd7d250885ca67e24c2851bb52ed067c62cea4f317f8431577569c7f3dc461c325f549c1eec
7
+ data.tar.gz: 5c9d7a190cd1afd1d3561c90ca7d1c5b1491ef886f4a5ddadfdfbfb2371ddcea46241758b7b328569e7d105f400364131258eb65bef389f20f2aac819605d1dc
data/.circleci/config.yml ADDED
@@ -0,0 +1,41 @@
1
+ version: 2
2
+ jobs:
3
+ build:
4
+ working_directory: ~/naginegi
5
+ docker:
6
+ - image: circleci/ruby:2.6.3
7
+ environment:
8
+ TZ: /usr/share/zoneinfo/Asia/Tokyo
9
+ steps:
10
+ - checkout
11
+
12
+ - restore_cache:
13
+ name: Restore bundle cache
14
+ key: naginegi-{{ checksum "Gemfile.lock" }}
15
+
16
+ - run:
17
+ name: Install Bundler
18
+ command: gem install bundler -v 2.1.4
19
+
20
+ - run:
21
+ name: Run bundle install
22
+ command: bundle install --path vendor/bundle
23
+
24
+ - save_cache:
25
+ name: Store bundle cache
26
+ key: naginegi-{{ checksum "Gemfile.lock" }}
27
+ paths:
28
+ - vendor/bundle
29
+
30
+ - run:
31
+ name: chmod +x ./lint.sh
32
+ command: chmod +x ./lint.sh
33
+
34
+ - run:
35
+ name: ./lint.sh
36
+ command: ./lint.sh
37
+
38
+ - run:
39
+ name: Run rspec
40
+ command: bundle exec rspec spec/
41
+
data/.gitignore ADDED
@@ -0,0 +1,41 @@
1
+ *.rbc
2
+ capybara-*.html
3
+ .rspec
4
+ .rspec_status
5
+ *.log
6
+ /log
7
+ /tmp
8
+ /db/*.sqlite3
9
+ /db/*.sqlite3-journal
10
+ /db/schema.rb
11
+ /public/system
12
+ /coverage/
13
+ /spec/tmp
14
+ *.orig
15
+ rerun.txt
16
+ pickle-email-*.html
17
+
18
+ .project
19
+
20
+ config/initializers/secret_token.rb
21
+
22
+ .env
23
+
24
+ /.bundle
25
+ /vendor/bundle
26
+
27
+ .rvmrc
28
+
29
+ /vendor/assets/bower_components
30
+ *.bowerrc
31
+ bower.json
32
+
33
+ .powenv
34
+
35
+ .byebug_history
36
+
37
+ .ruby-version
38
+
39
+ .DS_Store
40
+
41
+ *.gem
data/.rubocop.yml ADDED
@@ -0,0 +1,105 @@
1
+ AllCops:
2
+ TargetRubyVersion: 2.4.1
3
+ Include:
4
+ - '**/Gemfile'
5
+ - '**/Rakefile'
6
+ - '**/config.ru'
7
+ Exclude:
8
+ - 'db/**/*'
9
+ - 'config/**/*'
10
+ - 'script/**/*'
11
+ - 'vendor/**/*'
12
+ - 'bin/*'
13
+ - !ruby/regexp /old_and_unused\.rb#x2F;
14
+
15
+ AlignParameters:
16
+ Enabled: false
17
+
18
+ AsciiComments:
19
+ Enabled: false
20
+
21
+ BracesAroundHashParameters:
22
+ Enabled: false
23
+
24
+ Bundler/OrderedGems:
25
+ Enabled: false
26
+
27
+ ClassAndModuleChildren:
28
+ Enabled: false
29
+
30
+ ClassLength:
31
+ Enabled: false
32
+
33
+ Layout/AlignHash:
34
+ Enabled: false
35
+
36
+ Layout/EndOfLine:
37
+ Enabled: false
38
+
39
+ Layout/IndentHash:
40
+ Enabled: false
41
+
42
+ Layout/MultilineArrayBraceLayout:
43
+ Enabled: false
44
+
45
+ Layout/MultilineHashBraceLayout:
46
+ Enabled: false
47
+
48
+ Layout/MultilineMethodCallIndentation:
49
+ Enabled: false
50
+
51
+ Metrics/BlockLength:
52
+ Max: 50
53
+ Exclude:
54
+ - 'spec/**/*'
55
+ - 'config/routes.rb'
56
+ - 'app/jobs/**/*.rb'
57
+ - 'lib/tasks/**/*.rb'
58
+
59
+ Metrics/AbcSize:
60
+ Enabled: false
61
+
62
+ Metrics/CyclomaticComplexity:
63
+ Enabled: false
64
+
65
+ Metrics/LineLength:
66
+ Enabled: false
67
+
68
+ Metrics/MethodLength:
69
+ Enabled: false
70
+
71
+ Metrics/PerceivedComplexity:
72
+ Enabled: false
73
+
74
+ ModuleLength:
75
+ Enabled: false
76
+
77
+ Style/ClassAndModuleCamelCase:
78
+ Enabled: false
79
+
80
+ Style/Documentation:
81
+ Enabled: false
82
+
83
+ Style/FrozenStringLiteralComment:
84
+ Enabled: false
85
+
86
+ Style/GuardClause:
87
+ Enabled: false
88
+
89
+ Style/Next:
90
+ Enabled: false
91
+
92
+ Style/NumericLiterals:
93
+ Enabled: false
94
+
95
+ Style/RaiseArgs:
96
+ Enabled: false
97
+
98
+ Style/RedundantBegin:
99
+ Enabled: false
100
+
101
+ Style/Lambda:
102
+ Enabled: false
103
+
104
+ Rails/SkipsModelValidations:
105
+ Enabled: false
data/.travis.yml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ sudo: false
3
+ language: ruby
4
+ cache: bundler
5
+ rvm:
6
+ - 2.6.3
7
+ before_install: gem install bundler -v 2.0.2
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,120 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ naginegi (0.1.0)
5
+ google-cloud-bigquery (= 1.18.1)
6
+ mysql2 (= 0.5.3)
7
+ mysql2-cs-bind (= 0.0.7)
8
+ pg (= 1.2.2)
9
+ unindent (= 1.0)
10
+
11
+ GEM
12
+ remote: https://rubygems.org/
13
+ specs:
14
+ addressable (2.7.0)
15
+ public_suffix (>= 2.0.2, < 5.0)
16
+ ast (2.4.0)
17
+ concurrent-ruby (1.1.5)
18
+ declarative (0.0.10)
19
+ declarative-option (0.1.0)
20
+ diff-lcs (1.3)
21
+ faraday (0.17.3)
22
+ multipart-post (>= 1.2, < 3)
23
+ google-api-client (0.36.4)
24
+ addressable (~> 2.5, >= 2.5.1)
25
+ googleauth (~> 0.9)
26
+ httpclient (>= 2.8.1, < 3.0)
27
+ mini_mime (~> 1.0)
28
+ representable (~> 3.0)
29
+ retriable (>= 2.0, < 4.0)
30
+ signet (~> 0.12)
31
+ google-cloud-bigquery (1.18.1)
32
+ concurrent-ruby (~> 1.0)
33
+ google-api-client (~> 0.33)
34
+ google-cloud-core (~> 1.2)
35
+ googleauth (~> 0.9)
36
+ mini_mime (~> 1.0)
37
+ google-cloud-core (1.5.0)
38
+ google-cloud-env (~> 1.0)
39
+ google-cloud-errors (~> 1.0)
40
+ google-cloud-env (1.3.0)
41
+ faraday (~> 0.11)
42
+ google-cloud-errors (1.0.0)
43
+ googleauth (0.10.0)
44
+ faraday (~> 0.12)
45
+ jwt (>= 1.4, < 3.0)
46
+ memoist (~> 0.16)
47
+ multi_json (~> 1.11)
48
+ os (>= 0.9, < 2.0)
49
+ signet (~> 0.12)
50
+ httpclient (2.8.3)
51
+ jwt (2.2.1)
52
+ memoist (0.16.2)
53
+ mini_mime (1.0.2)
54
+ multi_json (1.14.1)
55
+ multipart-post (2.1.1)
56
+ mysql2 (0.5.3)
57
+ mysql2 (0.5.3-x64-mingw32)
58
+ mysql2-cs-bind (0.0.7)
59
+ mysql2
60
+ os (1.0.1)
61
+ parallel (1.19.1)
62
+ parser (2.7.0.1)
63
+ ast (~> 2.4.0)
64
+ pg (1.2.2)
65
+ pg (1.2.2-x64-mingw32)
66
+ powerpack (0.1.2)
67
+ public_suffix (4.0.3)
68
+ rainbow (2.2.2)
69
+ rake
70
+ rake (12.3.3)
71
+ representable (3.0.4)
72
+ declarative (< 0.1.0)
73
+ declarative-option (< 0.2.0)
74
+ uber (< 0.2.0)
75
+ retriable (3.1.2)
76
+ rspec (3.8.0)
77
+ rspec-core (~> 3.8.0)
78
+ rspec-expectations (~> 3.8.0)
79
+ rspec-mocks (~> 3.8.0)
80
+ rspec-core (3.8.2)
81
+ rspec-support (~> 3.8.0)
82
+ rspec-expectations (3.8.6)
83
+ diff-lcs (>= 1.2.0, < 2.0)
84
+ rspec-support (~> 3.8.0)
85
+ rspec-mocks (3.8.2)
86
+ diff-lcs (>= 1.2.0, < 2.0)
87
+ rspec-support (~> 3.8.0)
88
+ rspec-support (3.8.3)
89
+ rubocop (0.49.1)
90
+ parallel (~> 1.10)
91
+ parser (>= 2.3.3.1, < 3.0)
92
+ powerpack (~> 0.1)
93
+ rainbow (>= 1.99.1, < 3.0)
94
+ ruby-progressbar (~> 1.7)
95
+ unicode-display_width (~> 1.0, >= 1.0.1)
96
+ ruby-progressbar (1.10.1)
97
+ signet (0.12.0)
98
+ addressable (~> 2.3)
99
+ faraday (~> 0.9)
100
+ jwt (>= 1.5, < 3.0)
101
+ multi_json (~> 1.10)
102
+ timecop (0.9.1)
103
+ uber (0.1.0)
104
+ unicode-display_width (1.6.0)
105
+ unindent (1.0)
106
+
107
+ PLATFORMS
108
+ ruby
109
+ x64-mingw32
110
+
111
+ DEPENDENCIES
112
+ bundler (= 2.1.4)
113
+ naginegi!
114
+ rake (= 12.3.3)
115
+ rspec (= 3.8.0)
116
+ rubocop (= 0.49.1)
117
+ timecop (= 0.9.1)
118
+
119
+ BUNDLED WITH
120
+ 2.1.4
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2019 cobot00
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,189 @@
1
+ # Naginegi
2
+
3
+ Generate Embulk config and BigQuery schema from MySQL and PostgreSQL schema and run Embulk.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'naginegi'
11
+ ```
12
+
13
+ And then execute:
14
+
15
+ $ bundle
16
+
17
+ Or install it yourself as:
18
+
19
+ $ gem install naginegi
20
+
21
+ ## Embulk setup
22
+ `Naginegi` is utility for `Embulk` .
23
+ You need to install `Embulk` and install some gems like below.
24
+
25
+ ```bash
26
+ embulk gem install embulk-input-mysql --version 0.10.1
27
+ embulk gem install embulk-input-postgresql --version 0.10.1
28
+ embulk gem install embulk-output-bigquery --version 0.6.4
29
+ embulk gem install embulk-parser-jsonl --version 0.2.0
30
+ embulk gem install embulk-formatter-jsonl --version 0.1.4
31
+ ```
32
+
33
+ ## Usage
34
+ Require `database.yml` and `table.yml`.
35
+ `database.yml` requires `db_type` (mysql or postgresql).
36
+
37
+ |RDBMS|db_type|
38
+ |---|---|
39
+ |MySQL|mysql|
40
+ |PostgreSQL|postgresql|
41
+
42
+ Below is a sample config file.
43
+
44
+ ### database.yml
45
+ ```yml
46
+ db01:
47
+ db_type: mysql
48
+ host: localhost
49
+ username: root
50
+ password: pswd
51
+ database: production
52
+ bq_dataset: mysql
53
+
54
+ db02:
55
+ db_type: postgresql
56
+ host: localhost
57
+ username: root
58
+ password: pswd
59
+ database: production
60
+ bq_dataset: pg
61
+
62
+ ```
63
+
64
+ **Caution: Embulk doesn't allow no password for MySQL**
65
+
66
+ ### table.yml
67
+ ```yml
68
+ db01:
69
+ tables:
70
+ - name: users
71
+ - name: events
72
+ - name: hobbies
73
+
74
+ db02:
75
+ tables:
76
+ - name: administrators
77
+ - name: configs
78
+ ```
79
+
80
+ Naginegi requires BigQuery parameters like below.
81
+
82
+ ```ruby
83
+ [sample.rb]
84
+ require 'naginegi'
85
+
86
+ config = {
87
+ 'project_id' => 'BIGQUERY_PROJECT_ID',
88
+ 'service_email' => 'SERVICE_ACCOUNT_EMAIL',
89
+ 'auth_method' => 'json_key',
90
+ 'json_keyfile' => 'JSON_KEYFILE_PATH',
91
+ 'schema_dir' => '/var/tmp/embulk/schema',
92
+ 'config_dir' => '/var/tmp/embulk/config'
93
+ }
94
+
95
+ client = Naginegi::EmbulkClient.new
96
+ client.generate_config(config)
97
+ client.run(config)
98
+ ```
99
+
100
+ ```bash
101
+ ruby sample.rb
102
+ ```
103
+
104
+ ## Features
105
+ ### process status
106
+ `Naginegi` returns process status as boolean.
107
+ If all tables are succeed, then returns `true`, else `false` .
108
+ It is useful to control system flow.
109
+
110
+ ```ruby
111
+ process_status = Naginegi::EmbulkClient.new.run(config)
112
+ exit 1 unless process_status
113
+ ```
114
+
115
+ ### narrow tables
116
+ You can narrow actual target tables from `table.yml` for test or to retry.
117
+ If no target tables is given, `Naginegi` will execute all tables.
118
+
119
+ ```ruby
120
+ # in case, all tables are ['users', 'purchases', 'items']
121
+ target_tables = ['users', 'purchases']
122
+ Naginegi::EmbulkClient.new.run(config, target_tables)
123
+ ```
124
+
125
+ ### retry
126
+ You can set retry count.
127
+ If any table failed, only failed table will be retried until retry count.
128
+ If no retry count is given, `Naginegi` dosen't retry.
129
+
130
+ ```ruby
131
+ # 2 times retry will execute
132
+ Naginegi::EmbulkClient.new.run(config, [], 2)
133
+ ```
134
+
135
+ ### SQL condition
136
+ If you set `condition` to a table in `table.yml` , SQL is generated like below.
137
+ It is useful for large size table.
138
+
139
+ ```yml
140
+ [table.yml]
141
+ production:
142
+ tables:
143
+ - name: users
144
+ - name: events
145
+ conditon: created_at < CURRENT_DATE()
146
+ ```
147
+
148
+ ```sql
149
+ SELECT * FROM users
150
+ SELECT * FROM events WHERE created_at < CURRENT_DATE()
151
+ ```
152
+
153
+ ### daily snapshot
154
+ BigQuery supports table wildcard expression of a specific set of daily tables, for example, `sales20150701` .
155
+ If you need daily snapshot of a table for BigQuery, use `daily_snapshot` option to `database.yml` or `table.yml` like below.
156
+ `daily_snapshot` option effects all tables in case of `database.yml` .
157
+ On the other hand, only target table in `table.yml` .
158
+ **Daily part is determined by execute date.**
159
+
160
+ ```yml
161
+ [database.yml]
162
+ production:
163
+ host: localhost
164
+ username: root
165
+ password: pswd
166
+ database: production
167
+ bq_dataset: mysql
168
+ daily_snapshot: true
169
+ ```
170
+
171
+ ```yml
172
+ [table.yml]
173
+ production:
174
+ tables:
175
+ - name: users
176
+ - name: events
177
+ daily_snapshot: true
178
+ - name: hobbies
179
+
180
+ Only `events` is renamed to `eventsYYYYMMDD` for BigQuery.
181
+ ```
182
+
183
+ ## Contributing
184
+
185
+ 1. Fork it ( https://github.com/[my-github-username]/naginegi/fork )
186
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
187
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
188
+ 4. Push to the branch (`git push origin my-new-feature`)
189
+ 5. Create a new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rspec/core/rake_task'
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "naginegi"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=#x27;\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
data/lib/naginegi.rb ADDED
@@ -0,0 +1,57 @@
1
+ require 'naginegi/version'
2
+ require 'naginegi/embulk_config'
3
+ require 'naginegi/embulk'
4
+ require 'naginegi/mysql'
5
+ require 'naginegi/postgresql'
6
+ require 'logger'
7
+
8
+ module Naginegi
9
+ class EmbulkRunner
10
+ def initialize
11
+ @logger = Logger.new(STDOUT)
12
+ @logger.datetime_format = '%Y-%m-%d %H:%M:%S'
13
+ end
14
+
15
+ def generate_config(bq_config)
16
+ Naginegi::EmbulkConfig.new.generate_config(db_configs, bq_config)
17
+ end
18
+
19
+ def run(bq_config, target_table_names = [], retry_max = 0)
20
+ cmd = 'embulk --version'
21
+ unless system(cmd)
22
+ @logger.error('Cannot execute Embulk!!')
23
+ @logger.error('Cofirm Embulk install and environment')
24
+ return
25
+ end
26
+
27
+ error_tables = run_and_retry(bq_config, target_table_names, retry_max, 0)
28
+ error_tables.empty?
29
+ end
30
+
31
+ private
32
+
33
+ def run_and_retry(bq_config, target_table_names, retry_max, retry_count)
34
+ error_tables = Naginegi::Embulk.new.run(
35
+ db_configs,
36
+ table_configs,
37
+ bq_config,
38
+ target_table_names
39
+ )
40
+ if !error_tables.empty? && retry_count < retry_max
41
+ @logger.warn('------------------------------------')
42
+ @logger.warn("retry start -> #{retry_count + 1} time")
43
+ @logger.warn('------------------------------------')
44
+ error_tables = run_and_retry(bq_config, error_tables, retry_max, retry_count + 1)
45
+ end
46
+ error_tables
47
+ end
48
+
49
+ def db_configs
50
+ @db_configs ||= YAML.load_file('database.yml')
51
+ end
52
+
53
+ def table_configs
54
+ @table_configs ||= Naginegi::MySQL::TableConfig.generate_table_configs
55
+ end
56
+ end
57
+ end
data/lib/naginegi/bigquery.rb ADDED
@@ -0,0 +1,96 @@
1
+ require 'json'
2
+ require 'erb'
3
+ require 'google/cloud/bigquery'
4
+ require 'unindent'
5
+ require 'date'
6
+
7
+ module Naginegi
8
+ class BigQuery
9
+ CONTENTS = <<-EOS.unindent
10
+ in:
11
+ type: <%= db_type %>
12
+ user: <%= user %>
13
+ password: <%= password %>
14
+ database: <%= database %>
15
+ host: <%= host %>
16
+ query: |
17
+ <%= query %>
18
+ <%= options %>
19
+ out:
20
+ type: bigquery
21
+ auth_method: <%= auth_method %>
22
+ json_keyfile: <%= json_keyfile %>
23
+ project: <%= project %>
24
+ service_account_email: <%= service_account_email %>
25
+ dataset: <%= dataset %>
26
+ table: <%= table_name %>
27
+ schema_file: <%= schema_file %>
28
+ auto_create_table: true
29
+ path_prefix: <%= path_prefix %>
30
+ source_format: NEWLINE_DELIMITED_JSON
31
+ file_ext: .json.gz
32
+ delete_from_local_when_job_end: 1
33
+ formatter:
34
+ type: jsonl
35
+ encoders:
36
+ - {type: gzip}
37
+ EOS
38
+
39
+ def initialize(config)
40
+ @config = config.dup
41
+ @current_date = Date.today
42
+ end
43
+
44
+ def self.generate_schema(columns)
45
+ json_body = columns.map(&:to_json).join(",\n")
46
+ "[\n" + json_body + "\n]\n"
47
+ end
48
+
49
+ def self.generate_sql(table_config, columns)
50
+ columns = columns.map(&:converted_value)
51
+ sql = "SELECT #{columns.join(',')}"
52
+ sql << " FROM #{table_config.name}"
53
+ sql << " WHERE #{table_config.condition}" if table_config.condition
54
+ sql << "\n"
55
+ sql
56
+ end
57
+
58
+ def generate_embulk_config(db_name, db_config, table_config, columns)
59
+ db_type = db_config['db_type']
60
+ host = db_config['host']
61
+ user = db_config['username']
62
+ password = db_config['password']
63
+ database = db_config['database']
64
+ options = if db_type == 'mysql'
65
+ "options: {useLegacyDatetimeCode: false, serverTimezone: #{db_config['timezone']}}"
66
+ else
67
+ ''
68
+ end
69
+ query = Naginegi::BigQuery.generate_sql(table_config, columns)
70
+
71
+ auth_method = @config['auth_method']
72
+ json_keyfile = @config['json_keyfile']
73
+ project = @config['project_id']
74
+ service_account_email = @config['service_email']
75
+ dataset = db_config['bq_dataset']
76
+ table_name = actual_table_name(table_config.name, db_config['daily_snapshot'] || table_config.daily_snapshot)
77
+ schema_file = "#{@config['schema_dir']}/#{db_name}/#{table_config.name}.json"
78
+ path_prefix = "/var/tmp/embulk_#{db_name}_#{table_config.name}"
79
+
80
+ ERB.new(CONTENTS).result(binding)
81
+ end
82
+
83
+ def delete_table(dataset, table_name)
84
+ bq = Google::Cloud::Bigquery.new(
85
+ project: @config['project_id'],
86
+ keyfile: @config['json_keyfile']
87
+ )
88
+ bq.service.delete_table(dataset, table_name)
89
+ end
90
+
91
+ def actual_table_name(table_name, daily_snapshot)
92
+ return table_name unless daily_snapshot
93
+ table_name + @current_date.strftime('%Y%m%d')
94
+ end
95
+ end
96
+ end
data/lib/naginegi/embulk.rb ADDED
@@ -0,0 +1,72 @@
1
+ require 'logger'
2
+
3
+ module Naginegi
4
+ class Embulk
5
+ def initialize
6
+ @logger = Logger.new(STDOUT)
7
+ @logger.datetime_format = '%Y-%m-%d %H:%M:%S'
8
+ end
9
+
10
+ def run(db_configs, all_table_configs, bq_config, target_table_names = [])
11
+ error_tables = []
12
+ db_configs.keys.each do |db_name|
13
+ table_configs = select_table_configs(all_table_configs[db_name], target_table_names)
14
+ error_tables += run_by_database(
15
+ db_name,
16
+ table_configs,
17
+ bq_config,
18
+ db_configs[db_name]['bq_dataset']
19
+ )
20
+ end
21
+ error_tables
22
+ end
23
+
24
+ def select_table_configs(table_configs, target_table_names)
25
+ return table_configs if target_table_names.empty?
26
+ table_configs.select { |table_config| target_table_names.include?(table_config.name) }
27
+ end
28
+
29
+ private
30
+
31
+ def run_by_database(db_name, table_configs, bq_config, bq_dataset)
32
+ process_times = []
33
+ error_tables = []
34
+
35
+ bq_utility = Naginegi::BigQuery.new(bq_config)
36
+
37
+ table_configs.each do |table_config|
38
+ start_time = Time.now
39
+ @logger.info("table: #{table_config.name} - start")
40
+
41
+ begin
42
+ bq_utility.delete_table(bq_dataset, table_config.name)
43
+ @logger.info("#{table_config.name} is deleted")
44
+ rescue => e
45
+ @logger.warn(e.message)
46
+ end
47
+
48
+ cmd = "embulk run #{bq_config['config_dir']}/#{db_name}/#{table_config.name}.yml"
49
+ @logger.info("cmd: #{cmd}")
50
+
51
+ if system(cmd)
52
+ result = 'success'
53
+ else
54
+ result = 'error'
55
+ error_tables << table_config.name
56
+ end
57
+
58
+ process_time = "table: #{table_config.name} - result: #{result} #{format('%10.1f', Time.now - start_time)}sec"
59
+ @logger.info(process_time)
60
+
61
+ process_times << process_time
62
+ end
63
+
64
+ @logger.info('------------------------------------')
65
+ @logger.info("db_name: #{db_name}")
66
+
67
+ process_times.each { |process_time| @logger.info(process_time) }
68
+
69
+ error_tables
70
+ end
71
+ end
72
+ end
data/lib/naginegi/embulk_config.rb ADDED
@@ -0,0 +1,49 @@
1
+ module Naginegi
2
+ class EmbulkConfig
3
+ def generate_config(db_configs, bq_config)
4
+ bq_utility = BigQuery.new(bq_config)
5
+
6
+ db_configs.keys.each do |db_name|
7
+ db_config = db_configs[db_name]
8
+ table_configs = all_table_configs[db_name]
9
+ db_type = db_config['db_type']
10
+
11
+ case db_type
12
+ when 'mysql'
13
+ sql_client = MySQL::MySQLClient.new(db_config)
14
+ when 'postgresql'
15
+ sql_client = PostgreSQL::PgClient.new(db_config)
16
+ end
17
+
18
+ table_configs.each do |table_config|
19
+ write(
20
+ "#{bq_config['schema_dir']}/#{db_name}",
21
+ "#{table_config.name}.json",
22
+ sql_client.generate_bq_schema(table_config.name)
23
+ )
24
+ write(
25
+ "#{bq_config['config_dir']}/#{db_name}",
26
+ "#{table_config.name}.yml",
27
+ bq_utility.generate_embulk_config(
28
+ db_name,
29
+ db_config,
30
+ table_config,
31
+ sql_client.columns(table_config.name)
32
+ )
33
+ )
34
+ end
35
+ end
36
+ end
37
+
38
+ private
39
+
40
+ def write(directory, file_name, content)
41
+ FileUtils.mkdir_p(directory) unless FileTest.exist?(directory)
42
+ File.write("#{directory}/#{file_name}", content)
43
+ end
44
+
45
+ def all_table_configs
46
+ @all_table_configs ||= MySQL::TableConfig.generate_table_configs
47
+ end
48
+ end
49
+ end
data/lib/naginegi/mysql.rb ADDED
@@ -0,0 +1,119 @@
1
+ require 'mysql2-cs-bind'
2
+ require 'json'
3
+ require 'yaml'
4
+ require 'fileutils'
5
+ require 'naginegi/bigquery'
6
+
7
+ module Naginegi
8
+ module MySQL
9
+ class MySQLClient
10
+ COLUMN_SQL = <<-SQL.freeze
11
+ SELECT column_name, data_type
12
+ FROM INFORMATION_SCHEMA.COLUMNS
13
+ WHERE table_schema = ?
14
+ AND table_name = ?
15
+ ORDER BY ordinal_position
16
+ SQL
17
+
18
+ def initialize(database_config)
19
+ @database_config = database_config
20
+ end
21
+
22
+ def client
23
+ @client ||= Mysql2::Client.new(
24
+ host: @database_config['host'],
25
+ username: @database_config['username'],
26
+ password: @database_config['password'],
27
+ database: @database_config['database']
28
+ )
29
+ end
30
+
31
+ def generate_bq_schema(table_name)
32
+ infos = columns(table_name)
33
+ BigQuery.generate_schema(infos)
34
+ end
35
+
36
+ def columns(table_name)
37
+ rows = client.xquery(COLUMN_SQL, @database_config['database'], table_name)
38
+ rows.map { |row| Column.new(row['column_name'], row['data_type']) }
39
+ end
40
+ end
41
+
42
+ class TableConfig
43
+ attr_reader :name, :daily_snapshot, :condition
44
+
45
+ def initialize(config)
46
+ @name = config['name']
47
+ @daily_snapshot = config['daily_snapshot'] || false
48
+ @condition = config['condition']
49
+ end
50
+
51
+ def self.generate_table_configs(file_path = 'table.yml')
52
+ configs = YAML.load_file(file_path)
53
+ configs.each_with_object({}) do |(db, database_config), table_configs|
54
+ table_configs[db] = database_config['tables'].map { |config| TableConfig.new(config) }
55
+ table_configs
56
+ end
57
+ end
58
+
59
+ def ==(other)
60
+ instance_variables.all? do |v|
61
+ instance_variable_get(v) == other.instance_variable_get(v)
62
+ end
63
+ end
64
+ end
65
+
66
+ class Column
67
+ attr_reader :column_name, :data_type
68
+
69
+ TYPE_MAPPINGS = {
70
+ 'int' => 'INT64',
71
+ 'tinyint' => 'INT64',
72
+ 'smallint' => 'INT64',
73
+ 'mediumint' => 'INT64',
74
+ 'bigint' => 'INT64',
75
+ 'float' => 'FLOAT64',
76
+ 'double' => 'FLOAT64',
77
+ 'decimal' => 'FLOAT64',
78
+ 'char' => 'STRING',
79
+ 'varchar' => 'STRING',
80
+ 'tinytext' => 'STRING',
81
+ 'text' => 'STRING',
82
+ 'date' => 'TIMESTAMP',
83
+ 'datetime' => 'TIMESTAMP',
84
+ 'timestamp' => 'TIMESTAMP'
85
+ }.freeze
86
+
87
+ def initialize(column_name, data_type)
88
+ @column_name = column_name
89
+ @data_type = data_type
90
+ end
91
+
92
+ def bigquery_data_type
93
+ TYPE_MAPPINGS[@data_type]
94
+ end
95
+
96
+ def converted_value
97
+ if bigquery_data_type == 'TIMESTAMP'
98
+ # time zone translate to UTC
99
+ "UNIX_TIMESTAMP(#{escaped_column_name}) AS #{escaped_column_name}"
100
+ elsif data_type == 'tinyint'
101
+ # for MySQL tinyint(1) problem
102
+ "CAST(#{escaped_column_name} AS signed) AS #{escaped_column_name}"
103
+ else
104
+ escaped_column_name
105
+ end
106
+ end
107
+
108
+ def to_json(*a)
109
+ { 'name' => @column_name, 'type' => bigquery_data_type }.to_json(*a)
110
+ end
111
+
112
+ private
113
+
114
+ def escaped_column_name
115
+ "`#{@column_name}`"
116
+ end
117
+ end
118
+ end
119
+ end
data/lib/naginegi/postgresql.rb ADDED
@@ -0,0 +1,117 @@
1
+ require 'pg'
2
+ require 'json'
3
+ require 'yaml'
4
+ require 'fileutils'
5
+ require 'naginegi/bigquery'
6
+
7
+ module Naginegi
8
+ module PostgreSQL
9
+ class PgClient
10
+ COLUMN_SQL = <<-SQL.freeze
11
+ SELECT column_name, data_type
12
+ FROM INFORMATION_SCHEMA.COLUMNS
13
+ WHERE table_name = $1
14
+ ORDER BY ordinal_position
15
+ SQL
16
+
17
+ def initialize(db_config)
18
+ @db_config = db_config
19
+ end
20
+
21
+ def client
22
+ @client ||= PG::Connection.new(
23
+ host: @db_config['host'],
24
+ user: @db_config['username'],
25
+ password: @db_config['password'],
26
+ dbname: @db_config['database']
27
+ )
28
+ end
29
+
30
+ def generate_bq_schema(table_name)
31
+ infos = columns(table_name)
32
+ BigQuery.generate_schema(infos)
33
+ end
34
+
35
+ def columns(table_name)
36
+ rows = client.exec_params(COLUMN_SQL, [table_name])
37
+ rows.map { |row| Column.new(row['column_name'], row['data_type']) }
38
+ end
39
+ end
40
+
41
+ class TableConfig
42
+ attr_reader :name, :daily_snapshot, :condition
43
+
44
+ def initialize(config)
45
+ @name = config['name']
46
+ @daily_snapshot = config['daily_snapshot'] || false
47
+ @condition = config['condition']
48
+ end
49
+
50
+ def self.generate_table_configs(file_path = 'table.yml')
51
+ configs = YAML.load_file(file_path)
52
+ configs.each_with_object({}) do |(db, db_config), table_configs|
53
+ table_configs[db] = db_config['tables'].map { |config| TableConfig.new(config) }
54
+ table_configs
55
+ end
56
+ end
57
+
58
+ def ==(other)
59
+ instance_variables.all? do |v|
60
+ instance_variable_get(v) == other.instance_variable_get(v)
61
+ end
62
+ end
63
+ end
64
+
65
+ class Column
66
+ attr_reader :column_name, :data_type
67
+
68
+ TYPE_MAPPINGS = {
69
+ 'smallint' => 'INT64',
70
+ 'integer' => 'INT64',
71
+ 'bigint' => 'INT64',
72
+ 'smallserial' => 'INT64',
73
+ 'serial' => 'INT64',
74
+ 'bigserial' => 'INT64',
75
+ 'decimal' => 'FLOAT64',
76
+ 'numeric' => 'FLOAT64',
77
+ 'real' => 'FLOAT64',
78
+ 'double precision' => 'FLOAT64',
79
+ 'character' => 'STRING',
80
+ 'character varying' => 'STRING',
81
+ 'text' => 'STRING',
82
+ 'date' => 'TIMESTAMP',
83
+ 'timestamp' => 'TIMESTAMP',
84
+ 'timestamp with time zone' => 'TIMESTAMP',
85
+ 'boolean' => 'BOOL'
86
+ }.freeze
87
+
88
+ def initialize(column_name, data_type)
89
+ @column_name = column_name
90
+ @data_type = data_type
91
+ end
92
+
93
+ def bigquery_data_type
94
+ TYPE_MAPPINGS[@data_type]
95
+ end
96
+
97
+ def converted_value
98
+ if bigquery_data_type == 'TIMESTAMP'
99
+ # time zone translate to UTC
100
+ "EXTRACT(EPOCH FROM #{escaped_column_name}) AS #{escaped_column_name}"
101
+ else
102
+ escaped_column_name
103
+ end
104
+ end
105
+
106
+ def to_json(*a)
107
+ { 'name' => @column_name, 'type' => bigquery_data_type }.to_json(*a)
108
+ end
109
+
110
+ private
111
+
112
+ def escaped_column_name
113
+ "\"#{@column_name}\""
114
+ end
115
+ end
116
+ end
117
+ end
data/lib/naginegi/version.rb ADDED
@@ -0,0 +1,3 @@
1
+ module Naginegi
2
+ VERSION = '0.1.0'.freeze
3
+ end
data/lint.sh ADDED
@@ -0,0 +1,54 @@
1
+ #!/bin/bash
2
+
3
+ echo 'CIRCLE_BRANCH: ' ${CIRCLE_BRANCH}
4
+ TARGET_BRANCH=${CIRCLE_BRANCH}
5
+
6
+ # ローカルでの実行用にカレントブランチをセットする
7
+ if [ "$TARGET_BRANCH" = '' ]; then
8
+ TARGET_BRANCH=$(git rev-parse --abbrev-ref HEAD)
9
+ fi
10
+ echo 'TARGET_BRANCH: ' $TARGET_BRANCH
11
+
12
+ echo 'CIRCLE_BASE_BRANCH: ' ${CIRCLE_BASE_BRANCH}
13
+ BASE_BRANCH=${CIRCLE_BASE_BRANCH}
14
+
15
+ # ローカルでの実行時に環境変数をセットしていない時はmasterブランチと比較する
16
+ if [ "$BASE_BRANCH" = '' ]; then
17
+ BASE_BRANCH=origin/master
18
+ fi
19
+ echo 'BASE_BRANCH: ' $BASE_BRANCH
20
+
21
+ files=$(git diff --name-only $TARGET_BRANCH $BASE_BRANCH | grep -E '.rb' | egrep -v 'db/migrate|db/schema.rb|spec/factories')
22
+
23
+ error=false
24
+ for file in ${files}; do
25
+ # u-motion-api から同期されるファイルは確認不要
26
+ if [[ $file == *"app/models"* ]]; then
27
+ if [[ $file != *"app/models/admin"* ]]; then
28
+ continue
29
+ fi
30
+ fi
31
+
32
+ if [[ $file == *"app/serializers"* ]]; then
33
+ if [[ $file != *"app/serializers/admin"* ]]; then
34
+ continue
35
+ fi
36
+ fi
37
+
38
+ if [ -e $file ]; then
39
+ result=$(bundle exec rubocop ${file})
40
+ rubocop_error=$(echo "$result" | grep 'Offenses:')
41
+ if [ "$rubocop_error" != '' ]; then
42
+ error=true
43
+ echo ''
44
+ echo 'ERROR:' $file
45
+ echo "$result"
46
+ fi
47
+ fi
48
+ done
49
+
50
+ if $error; then
51
+ exit 1
52
+ fi
53
+
54
+ exit 0
data/naginegi.gemspec ADDED
@@ -0,0 +1,34 @@
1
+ lib = File.expand_path('lib', __dir__)
2
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3
+ require 'naginegi/version'
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = 'naginegi'
7
+ spec.version = Naginegi::VERSION
8
+ spec.authors = ['cobot00']
9
+ spec.email = ['kobori75@gmail.com']
10
+
11
+ spec.summary = %q{Embulk utility for MySQL and PostgreSQL to BigQuery}
12
+ spec.description = %q{Generate Embulk config and BigQuery schema from RDBMS schema}
13
+ spec.homepage = 'https://github.com/cobot00/naginegi'
14
+ spec.license = 'MIT'
15
+
16
+ spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
17
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
18
+ end
19
+ spec.bindir = 'exe'
20
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
21
+ spec.require_paths = ['lib']
22
+
23
+ spec.add_development_dependency 'bundler', '2.1.4'
24
+ spec.add_development_dependency 'rake', '12.3.3'
25
+ spec.add_development_dependency 'rspec', '3.8.0'
26
+ spec.add_development_dependency 'rubocop', '0.49.1'
27
+ spec.add_development_dependency 'timecop', '0.9.1'
28
+
29
+ spec.add_dependency 'google-cloud-bigquery', '1.18.1'
30
+ spec.add_dependency 'mysql2', '0.5.3'
31
+ spec.add_dependency 'mysql2-cs-bind', '0.0.7'
32
+ spec.add_dependency 'pg', '1.2.2'
33
+ spec.add_dependency 'unindent', '1.0'
34
+ end
metadata ADDED
@@ -0,0 +1,203 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: naginegi
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - cobot00
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2020-01-14 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '='
18
+ - !ruby/object:Gem::Version
19
+ version: 2.1.4
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - '='
25
+ - !ruby/object:Gem::Version
26
+ version: 2.1.4
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '='
32
+ - !ruby/object:Gem::Version
33
+ version: 12.3.3
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '='
39
+ - !ruby/object:Gem::Version
40
+ version: 12.3.3
41
+ - !ruby/object:Gem::Dependency
42
+ name: rspec
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - '='
46
+ - !ruby/object:Gem::Version
47
+ version: 3.8.0
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '='
53
+ - !ruby/object:Gem::Version
54
+ version: 3.8.0
55
+ - !ruby/object:Gem::Dependency
56
+ name: rubocop
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - '='
60
+ - !ruby/object:Gem::Version
61
+ version: 0.49.1
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '='
67
+ - !ruby/object:Gem::Version
68
+ version: 0.49.1
69
+ - !ruby/object:Gem::Dependency
70
+ name: timecop
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - '='
74
+ - !ruby/object:Gem::Version
75
+ version: 0.9.1
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - '='
81
+ - !ruby/object:Gem::Version
82
+ version: 0.9.1
83
+ - !ruby/object:Gem::Dependency
84
+ name: google-cloud-bigquery
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - '='
88
+ - !ruby/object:Gem::Version
89
+ version: 1.18.1
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - '='
95
+ - !ruby/object:Gem::Version
96
+ version: 1.18.1
97
+ - !ruby/object:Gem::Dependency
98
+ name: mysql2
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - '='
102
+ - !ruby/object:Gem::Version
103
+ version: 0.5.3
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - '='
109
+ - !ruby/object:Gem::Version
110
+ version: 0.5.3
111
+ - !ruby/object:Gem::Dependency
112
+ name: mysql2-cs-bind
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - '='
116
+ - !ruby/object:Gem::Version
117
+ version: 0.0.7
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - '='
123
+ - !ruby/object:Gem::Version
124
+ version: 0.0.7
125
+ - !ruby/object:Gem::Dependency
126
+ name: pg
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - '='
130
+ - !ruby/object:Gem::Version
131
+ version: 1.2.2
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - '='
137
+ - !ruby/object:Gem::Version
138
+ version: 1.2.2
139
+ - !ruby/object:Gem::Dependency
140
+ name: unindent
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - '='
144
+ - !ruby/object:Gem::Version
145
+ version: '1.0'
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - '='
151
+ - !ruby/object:Gem::Version
152
+ version: '1.0'
153
+ description: Generate Embulk config and BigQuery schema from RDBMS schema
154
+ email:
155
+ - kobori75@gmail.com
156
+ executables: []
157
+ extensions: []
158
+ extra_rdoc_files: []
159
+ files:
160
+ - ".circleci/config.yml"
161
+ - ".gitignore"
162
+ - ".rubocop.yml"
163
+ - ".travis.yml"
164
+ - Gemfile
165
+ - Gemfile.lock
166
+ - LICENSE
167
+ - README.md
168
+ - Rakefile
169
+ - bin/console
170
+ - bin/setup
171
+ - lib/naginegi.rb
172
+ - lib/naginegi/bigquery.rb
173
+ - lib/naginegi/embulk.rb
174
+ - lib/naginegi/embulk_config.rb
175
+ - lib/naginegi/mysql.rb
176
+ - lib/naginegi/postgresql.rb
177
+ - lib/naginegi/version.rb
178
+ - lint.sh
179
+ - naginegi.gemspec
180
+ homepage: https://github.com/cobot00/naginegi
181
+ licenses:
182
+ - MIT
183
+ metadata: {}
184
+ post_install_message:
185
+ rdoc_options: []
186
+ require_paths:
187
+ - lib
188
+ required_ruby_version: !ruby/object:Gem::Requirement
189
+ requirements:
190
+ - - ">="
191
+ - !ruby/object:Gem::Version
192
+ version: '0'
193
+ required_rubygems_version: !ruby/object:Gem::Requirement
194
+ requirements:
195
+ - - ">="
196
+ - !ruby/object:Gem::Version
197
+ version: '0'
198
+ requirements: []
199
+ rubygems_version: 3.0.6
200
+ signing_key:
201
+ specification_version: 4
202
+ summary: Embulk utility for MySQL and PostgreSQL to BigQuery
203
+ test_files: []