checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
---
2
2
SHA256:
3
- metadata.gz: bcfcef49dffcc6459e8138b6b2aa2c29ebbf78e9fb186b747c2e56a92750d0d1
4
- data.tar.gz: faba8a61381425fd43c9314aa863f937b1c9746b6ff5104a24c3c2a260b0aab6
3
+ metadata.gz: 63a1b730db853663d1a26cc59b1935a8176fb9aa0bde0be6e5761499a8127109
4
+ data.tar.gz: cae74a3c182b09e6c2a1ee34149388263428ddb3a9db5a8b7058de2d304b8544
5
5
SHA512:
6
- metadata.gz: 2e962c91d54e9156459fe98b1250b92c2d9394c1b64c002960f8b3b21f17c3789f157238fc6bd4bec23646c1f5ef84626fc2ecbb5a3dd5d14e6a45ca77f90f99
7
- data.tar.gz: 4a92c6de07b5d1509b97d2f2a73730cd35e298a903ab36baad6c115b57d03ea4e02e121b345f933bead9d8954bd6ba8b5501f40b921e662ba18aba9331225932
6
+ metadata.gz: e52a3a0aafc32e09c275157bfcb9325167be90b8a6ab4cda02ddeccc5577c25e8477c64492751b24b3a1222640ce147a311b01c598500b9f6b1a7a1b67f1b88d
7
+ data.tar.gz: d8ade264acf142b682e05ce90a4ca32464f76a301e2e2c0440b4bf52e30d9d22f8ca84f58d02024c48c06c36f726b3b0fc050505f77ab262928f47ae5399cbd5
data/.github/workflows/ruby.yml ADDED
@@ -0,0 +1,20 @@
1
+ name: Ruby
2
+
3
+ on: [push]
4
+
5
+ jobs:
6
+ build:
7
+
8
+ runs-on: ubuntu-latest
9
+
10
+ steps:
11
+ - uses: actions/checkout@v1
12
+ - name: Set up Ruby 2.6
13
+ uses: actions/setup-ruby@v1
14
+ with:
15
+ ruby-version: 2.6.x
16
+ - name: Build and test with Rake
17
+ run: |
18
+ gem install bundler
19
+ bundle install --jobs 4 --retry 3
20
+ bundle exec rake
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
PATH
2
2
remote: .
3
3
specs:
4
- eventboss (1.1.0)
4
+ eventboss (1.1.1)
5
5
aws-sdk-sns (>= 1.1.0)
6
6
aws-sdk-sqs (>= 1.3.0)
7
7
concurrent-ruby (~> 1.0, >= 1.0.5)
@@ -11,23 +11,23 @@ GEM
11
11
remote: https://rubygems.org/
12
12
specs:
13
13
aws-eventstream (1.0.3)
14
- aws-partitions (1.189.0)
15
- aws-sdk-core (3.59.0)
14
+ aws-partitions (1.206.0)
15
+ aws-sdk-core (3.64.0)
16
16
aws-eventstream (~> 1.0, >= 1.0.2)
17
17
aws-partitions (~> 1.0)
18
18
aws-sigv4 (~> 1.1)
19
19
jmespath (~> 1.0)
20
- aws-sdk-sns (1.18.0)
21
- aws-sdk-core (~> 3, >= 3.58.0)
20
+ aws-sdk-sns (1.19.0)
21
+ aws-sdk-core (~> 3, >= 3.61.1)
22
22
aws-sigv4 (~> 1.1)
23
- aws-sdk-sqs (1.18.0)
24
- aws-sdk-core (~> 3, >= 3.58.0)
23
+ aws-sdk-sqs (1.21.0)
24
+ aws-sdk-core (~> 3, >= 3.61.1)
25
25
aws-sigv4 (~> 1.1)
26
26
aws-sigv4 (1.1.0)
27
27
aws-eventstream (~> 1.0, >= 1.0.2)
28
28
concurrent-ruby (1.1.5)
29
29
diff-lcs (1.3)
30
- dotenv (2.7.4)
30
+ dotenv (2.7.5)
31
31
jmespath (1.4.0)
32
32
rake (12.3.1)
33
33
rspec (3.7.0)
data/README.md CHANGED
@@ -102,24 +102,29 @@ AWS_SNS_ENDPOINT=http://localhost:4575 # when using with localstack
102
102
AWS_SQS_ENDPOINT=http://localhost:4576 # when using with localstack
103
103
```
104
104
105
- ### Logging and error handling
106
- To have more verbose logging, set `log_level` in configuration (default is `info`).
105
+ Be aware that `eventbus:deadletter:reload` rake task won't load your configuration if you are not using ENVs
106
+ in non Rails app, although to make it work you can extend your `Rakefile` with:
107
-
108
- Logger is used as default error handler. There is Airbrake handler available, to use it ensure you have `airbrake` or `airbrake-ruby` gem and add it to error handlers stack:
109
107
110
108
```ruby
111
- Eventboss.configure do |config|
112
- config.error_handlers << Eventboss::ErrorHandlers::Airbrake.new
109
+ load File.join(Gem::Specification.find_by_name('eventboss').gem_dir, 'lib', 'tasks', 'eventboss.rake')
110
+
111
+ task :environment do
112
+ # Load your environment
113
+ # Example:
114
+ # require_relative 'config/application'
113
115
end
116
+
117
+ task 'eventboss:deadletter:reload': :environment
114
118
```
115
119
116
- ### Polling strategy
120
+ ### Logging and error handling
121
+ To have more verbose logging, set `log_level` in configuration (default is `info`).
117
122
118
- Default is `Eventboss::Polling::Basic`. See `eventboss/polling/*` for other options. The configuration should be a `lambda` like so:
123
+ Logger is used as default error handler. There is Airbrake handler available, to use it ensure you have `airbrake` or `airbrake-ruby` gem and add it to error handlers stack:
119
124
120
125
```ruby
121
126
Eventboss.configure do |config|
122
- config.polling_strategy = lambda { |queues| Eventboss::Polling::TimedRoundRobin.new(queues) }
127
+ config.error_handlers << Eventboss::ErrorHandlers::Airbrake.new
123
128
end
124
129
```
125
130
data/lib/eventboss.rb CHANGED
@@ -19,11 +19,8 @@ require 'eventboss/worker'
19
19
require 'eventboss/fetcher'
20
20
require 'eventboss/publisher'
21
21
require 'eventboss/sender'
22
- require 'eventboss/manager'
23
22
require 'eventboss/runner'
24
23
require 'eventboss/logger'
25
- require 'eventboss/polling/basic'
26
- require 'eventboss/polling/timed_round_robin'
27
24
require 'eventboss/extensions'
28
25
29
26
# For Rails use railtie, for plain Ruby apps use custom scripts loader
data/lib/eventboss/configuration.rb CHANGED
@@ -11,7 +11,6 @@ module Eventboss
11
11
:eventboss_account_id,
12
12
:aws_access_key_id,
13
13
:aws_secret_access_key,
14
- :polling_strategy,
15
14
:aws_sns_endpoint,
16
15
:aws_sqs_endpoint,
17
16
:sns_sqs_name_infix
@@ -81,12 +80,6 @@ module Eventboss
81
80
defined_or_default('aws_sns_endpoint') { ENV['AWS_SNS_ENDPOINT'] }
82
81
end
83
82
84
- def polling_strategy
85
- defined_or_default('polling_strategy') do
86
- lambda { |queues| Eventboss::Polling::Basic.new(queues) }
87
- end
88
- end
89
-
90
83
def sns_sqs_name_infix
91
84
defined_or_default('sns_sqs_name_infix') { ENV['EVENTBUS_SQS_SNS_NAME_INFIX'] || 'eventboss' }
92
85
end
data/lib/eventboss/manager.rb DELETED
@@ -1,116 +0,0 @@
1
- module Eventboss
2
- class Manager
3
- MIN_DISPATCH_INTERVAL = 0.1
4
-
5
- def initialize(fetcher, polling_strategy, executor, queue_listeners, concurrency, error_handlers)
6
- @fetcher = fetcher
7
- @polling_strategy = polling_strategy
8
- @max_processors = concurrency
9
- @busy_processors = Concurrent::AtomicFixnum.new(0)
10
- @executor = executor
11
- @queue_listeners = queue_listeners
12
- @error_handlers = Array(error_handlers)
13
- end
14
-
15
- def start
16
- Eventboss::Logger.debug('Starting dispatch loop...')
17
-
18
- dispatch_loop
19
- end
20
-
21
- private
22
-
23
- def running?
24
- @executor.running?
25
- end
26
-
27
- def dispatch_loop
28
- return unless running?
29
-
30
- Eventboss::Logger.debug('Posting task to executor')
31
-
32
- @executor.post { dispatch }
33
- end
34
-
35
- def dispatch
36
- return unless running?
37
-
38
- if ready <= 0 || (queue = @polling_strategy.next_queue).nil?
39
- return sleep(MIN_DISPATCH_INTERVAL)
40
- end
41
- dispatch_single_messages(queue)
42
- rescue => ex
43
- handle_dispatch_error(ex)
44
- ensure
45
- Eventboss::Logger.debug('Ensuring dispatch loop')
46
- dispatch_loop
47
- end
48
-
49
- def busy
50
- @busy_processors.value
51
- end
52
-
53
- def ready
54
- @max_processors - busy
55
- end
56
-
57
- def processor_done(processor)
58
- Eventboss::Logger.info("Success", processor.jid)
59
- @busy_processors.decrement
60
- end
61
-
62
- def processor_error(processor, exception)
63
- @error_handlers.each { |handler| handler.call(exception, processor) }
64
- @busy_processors.decrement
65
- end
66
-
67
- def assign(queue, sqs_msg)
68
- return unless running?
69
-
70
- @busy_processors.increment
71
- processor = @queue_listeners[queue].new
72
-
73
- Concurrent::Promise.execute(executor: @executor) do
74
- body = JSON.parse(sqs_msg.body) rescue sqs_msg.body
75
- Eventboss::Logger.info("Started", processor.jid)
76
- processor.receive(body)
77
- end.then do
78
- cleanup(processor)
79
- postpone_if_needed(queue, sqs_msg, processor) || delete_from_queue(queue, sqs_msg)
80
- processor_done(processor)
81
- end.rescue do |e|
82
- cleanup(processor)
83
- postpone_if_needed(queue, sqs_msg, processor)
84
- processor_error(processor, e)
85
- end
86
- end
87
-
88
- def cleanup(_processor)
89
- if defined?(ActiveRecord)
90
- ::ActiveRecord::Base.clear_active_connections!
91
- end
92
- end
93
-
94
- def delete_from_queue(queue, sqs_msg)
95
- @fetcher.delete(queue, sqs_msg)
96
- end
97
-
98
- def postpone_if_needed(queue, sqs_msg, processor)
99
- return false unless processor.postponed_by
100
- @fetcher.change_message_visibility(queue, sqs_msg, processor.postponed_by)
101
- rescue => error
102
- Eventboss::Logger.info("Could not postpone message #{error.message}", processor.jid)
103
- end
104
-
105
- def dispatch_single_messages(queue)
106
- messages = @fetcher.fetch(queue, ready)
107
- @polling_strategy.messages_found(queue, messages.size)
108
- messages.each { |message| assign(queue, message) }
109
- end
110
-
111
- def handle_dispatch_error(ex)
112
- Eventboss::Logger.error("Error dispatching #{ex.message}")
113
- Process.kill('USR1', Process.pid)
114
- end
115
- end
116
- end
data/lib/eventboss/polling/basic.rb DELETED
@@ -1,68 +0,0 @@
1
- module Eventboss
2
- module Polling
3
- class Basic
4
- PAUSE_AFTER_EMPTY = 2 # seconds
5
-
6
- def initialize(queues, timer = Time)
7
- @queues = queues.to_a
8
- @timer = timer
9
- @paused_until = @queues.each_with_object(Hash.new) do |queue, hash|
10
- hash[queue] = @timer.at(0)
11
- end
12
-
13
- reset_next_queue
14
- end
15
-
16
- def next_queue
17
- next_active_queue
18
- end
19
-
20
- def messages_found(queue, messages_count)
21
- if messages_count == 0
22
- pause(queue)
23
- else
24
- reset_next_queue
25
- end
26
- end
27
-
28
- def active_queues
29
- @queues.reject { |q, _| queue_paused?(q) }
30
- end
31
-
32
- private
33
-
34
- def next_active_queue
35
- reset_next_queue if queues_unpaused_since?
36
-
37
- size = @queues.length
38
- size.times do
39
- queue = @queues[@next_queue_index]
40
- @next_queue_index = (@next_queue_index + 1) % size
41
- return queue unless queue_paused?(queue)
42
- end
43
-
44
- nil
45
- end
46
-
47
- def queues_unpaused_since?
48
- last = @last_unpause_check
49
- now = @last_unpause_check = @timer.now
50
-
51
- last && @paused_until.values.any? { |t| t > last && t <= now }
52
- end
53
-
54
- def reset_next_queue
55
- @next_queue_index = 0
56
- end
57
-
58
- def queue_paused?(queue)
59
- @paused_until[queue] > @timer.now
60
- end
61
-
62
- def pause(queue)
63
- return unless PAUSE_AFTER_EMPTY > 0
64
- @paused_until[queue] = @timer.now + PAUSE_AFTER_EMPTY
65
- end
66
- end
67
- end
68
- end
data/lib/eventboss/polling/timed_round_robin.rb DELETED
@@ -1,42 +0,0 @@
1
- module Eventboss
2
- module Polling
3
- class TimedRoundRobin
4
- PAUSE_AFTER_EMPTY = 2 # seconds
5
-
6
- def initialize(queues, timer = Time)
7
- @queues = queues.to_a
8
- @timer = timer
9
- @next_queue_index = 0
10
- @paused_until = @queues.each_with_object(Hash.new) do |queue, hash|
11
- hash[queue] = @timer.at(0)
12
- end
13
- end
14
-
15
- def next_queue
16
- size = @queues.length
17
- size.times do
18
- queue = @queues[@next_queue_index]
19
- @next_queue_index = (@next_queue_index + 1) % size
20
- return queue unless queue_paused?(queue)
21
- end
22
-
23
- nil
24
- end
25
-
26
- def messages_found(queue, messages_count)
27
- pause(queue) if messages_count == 0
28
- end
29
-
30
- private
31
-
32
- def queue_paused?(queue)
33
- @paused_until[queue] > @timer.now
34
- end
35
-
36
- def pause(queue)
37
- return unless PAUSE_AFTER_EMPTY > 0
38
- @paused_until[queue] = @timer.now + PAUSE_AFTER_EMPTY
39
- end
40
- end
41
- end
42
- end
data/lib/eventboss/railtie.rb CHANGED
@@ -1,5 +1,9 @@
1
1
class Eventboss::Railtie < Rails::Railtie
2
2
rake_tasks do
3
3
load 'tasks/eventboss.rake'
4
+
5
+ # Load rails environment before executing reload.
6
+ # It makes sure to load configuration file.
7
+ task 'eventboss:deadletter:reload': :environment
4
8
end
5
9
end
data/lib/eventboss/runner.rb CHANGED
@@ -23,38 +23,6 @@ module Eventboss
23
23
end
24
24
end
25
25
26
- def start
27
- configuration = Eventboss.configuration
28
-
29
- queue_listeners = Eventboss::QueueListener.list
30
- Eventboss::Instrumentation.add(queue_listeners)
31
- polling_strategy = configuration.polling_strategy.call(queue_listeners.keys)
32
-
33
- fetcher = Eventboss::Fetcher.new(configuration)
34
- executor = Concurrent.global_io_executor
35
-
36
- manager = Eventboss::Manager.new(
37
- fetcher,
38
- polling_strategy,
39
- executor,
40
- queue_listeners,
41
- configuration.concurrency,
42
- configuration.error_handlers
43
- )
44
-
45
- manager.start
46
-
47
- self_read = setup_signals([:SIGTERM])
48
-
49
- begin
50
- handle_signals(self_read)
51
- rescue Interrupt
52
- executor.shutdown
53
- executor.wait_for_termination
54
- exit 0
55
- end
56
- end
57
-
58
26
private
59
27
60
28
def setup_signals(signals)
@@ -69,7 +37,6 @@ module Eventboss
69
37
self_read
70
38
end
71
39
72
-
73
40
def handle_signals(self_read, launcher)
74
41
while readable_io = IO.select([self_read])
75
42
signal = readable_io.first[0].gets.strip
data/lib/eventboss/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
module Eventboss
2
- VERSION = "1.1.0"
2
+ VERSION = "1.1.1"
3
3
end
data/lib/tasks/eventboss.rake CHANGED
@@ -3,7 +3,7 @@ require 'rake'
3
3
namespace :eventboss do
4
4
namespace :deadletter do
5
5
desc 'Reload deadletter queue'
6
- task :reload, [:event_name, :source_app, :max_messages] do |_, args|
6
+ task :reload, [:event_name, :source_app, :max_messages] do |task, args|
7
7
source_app = args[:source_app]
8
8
event_name = args[:event_name]
9
9
@@ -13,7 +13,7 @@ namespace :eventboss do
13
13
# Ensure we don't fetch more than 10 messages from SQS
14
14
batch_size = max_messages == 0 ? 10 : [10, max_messages].min
15
15
16
- abort 'At least event name should be passed as argument' unless event_name
16
+ abort "[#{task.name}] At least event name should be passed as argument" unless event_name
17
17
18
18
queue_name = [
19
19
Eventboss.configuration.eventboss_app_name,
@@ -21,14 +21,14 @@ namespace :eventboss do
21
21
source_app,
22
22
event_name,
23
23
Eventboss.env
24
- ].join('-')
24
+ ].compact.join('-')
25
+ puts "[#{task.name}] Reloading #{queue_name}-deadletter (max: #{ max_messages }, batch: #{ batch_size })"
25
26
queue = Eventboss::Queue.new("#{queue_name}-deadletter")
26
27
send_queue = Eventboss::Queue.new(queue_name)
27
28
28
- puts "Reloading deadletter (max: #{ max_messages }, batch: #{ batch_size })"
29
- puts " #{queue.url}"
30
- puts ' to'
29
+ puts "[#{task.name}] #{queue.url}"
30
+ puts "[#{task.name}] to"
31
+ puts "[#{task.name}] #{send_queue.url}"
31
- puts " #{send_queue.url}"
32
32
33
33
fetcher = Eventboss::Fetcher.new(Eventboss.configuration)
34
34
client = fetcher.client
@@ -38,7 +38,7 @@ namespace :eventboss do
38
38
break if messages.count.zero?
39
39
40
40
messages.each do |message|
41
- puts "Publishing message: #{message.body}"
41
+ puts "[#{task.name}] Publishing message: #{message.body}"
42
42
client.send_message(queue_url: send_queue.url, message_body: message.body)
43
43
fetcher.delete(queue, message)
44
44
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
--- !ruby/object:Gem::Specification
2
2
name: eventboss
3
3
version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.1.1
5
5
platform: ruby
6
6
authors:
7
7
- AirHelp
8
8
autorequire:
9
9
bindir: bin
10
10
cert_chain: []
11
- date: 2019-07-30 00:00:00.000000000 Z
11
+ date: 2019-08-27 00:00:00.000000000 Z
12
12
dependencies:
13
13
- !ruby/object:Gem::Dependency
14
14
name: concurrent-ruby
@@ -128,6 +128,7 @@ executables:
128
128
extensions: []
129
129
extra_rdoc_files: []
130
130
files:
131
+ - ".github/workflows/ruby.yml"
131
132
- ".gitignore"
132
133
- ".rspec"
133
134
- CHANGELOG.md
@@ -151,9 +152,6 @@ files:
151
152
- lib/eventboss/logger.rb
152
153
- lib/eventboss/logging.rb
153
154
- lib/eventboss/long_poller.rb
154
- - lib/eventboss/manager.rb
155
- - lib/eventboss/polling/basic.rb
156
- - lib/eventboss/polling/timed_round_robin.rb
157
155
- lib/eventboss/publisher.rb
158
156
- lib/eventboss/queue.rb
159
157
- lib/eventboss/queue_listener.rb