[gentoo-commits] proj/ag:master commit in: /, lib/

[gentoo-commits] proj/ag:master commit in: /, lib/

[Robin H. Johnson]

commit:     813cbffb98449a5fb5485766e6738d52a30e9f75
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 21 21:44:19 2015 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Sat Feb 21 21:44:19 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/ag.git;a=commit;h=813cbffb

Hotfix the Maildir handling to not need our --fix renaming of files.

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

---
 ag              |  1 +
 lib/hotfixes.rb | 12 ++++++++++++
 2 files changed, 13 insertions(+)

diff --git a/ag b/ag
index 20fce8f..742abee 100755
--- a/ag
+++ b/ag
@@ -11,6 +11,7 @@ require_relative 'lib/utils'
 require_relative 'lib/threading'
 require_relative 'lib/rendering'
 require_relative 'lib/storage'
+require_relative 'lib/hotfixes'
 
 $options = OpenStruct.new
 $options.action = nil

diff --git a/lib/hotfixes.rb b/lib/hotfixes.rb
new file mode 100644
index 0000000..2dc8fc9
--- /dev/null
+++ b/lib/hotfixes.rb
@@ -0,0 +1,12 @@
+# Hotfixes for other classes
+
+# If the INFO block contains multiple colons, @info will be wrong.
+class Maildir::Message
+  protected
+  # Sets dir, unique_name, and info based on the key
+  def parse_key(key)
+    @dir, filename = key.split(File::SEPARATOR)
+    @dir = @dir.to_sym
+    @unique_name, @info = filename.split(COLON, 2)
+  end
+end

[gentoo-commits] proj/ag:master commit in: /, lib/

[Alex Legler]

commit:     3aeff5af03460eea82b31c493c6bfa635f80cf72
Author:     Alex Legler <alex <AT> a3li <DOT> li>
AuthorDate: Mon Feb 23 00:05:05 2015 +0000
Commit:     Alex Legler <a3li <AT> gentoo <DOT> org>
CommitDate: Mon Feb 23 00:05:05 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/ag.git;a=commit;h=3aeff5af

Implement --info

---
 ag             | 23 ++++++++++++++++++++++-
 lib/storage.rb | 18 ++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/ag b/ag
index be04609..29ee0f2 100755
--- a/ag
+++ b/ag
@@ -42,6 +42,12 @@ op = OptionParser.new do |opts|
     $options.action = :do_delete
   end
 
+  opts.on('--info', 'Display message details. Needs --file, --msgid, or --hash') do
+    abort 'Can only select one action' if $options.action != nil
+
+    $options.action = :do_info
+  end
+
   opts.on('--reindex', 'Reindex message. Needs --file') do
     abort 'Can only select one action' if $options.action != nil
 
@@ -144,7 +150,22 @@ def do_reindex
 end
 
 def do_info
-  abort 'Come back later.'
+  id = Ag::Utils.resolve_id
+
+  begin
+    message = Ag::Storage.get($options.name, id)
+
+    raise 'No such message' unless message
+
+    require 'pp'
+    str = "Message #{id}"
+    $stderr.puts str
+    $stderr.puts '-' * str.length
+
+    pp message['_source']
+  rescue => e
+    $stderr.puts "Cannot display message: #{e}"
+  end
 end
 
 ###############################################################################

diff --git a/lib/storage.rb b/lib/storage.rb
index 37083fa..b4a518e 100644
--- a/lib/storage.rb
+++ b/lib/storage.rb
@@ -220,4 +220,22 @@ module Ag::Storage
 
     result['hits']['total']
   end
+  def get(list, id)
+    result = $es.search(
+      index: 'ml-' + list,
+      size: 1,
+      body: {
+        query: {
+          filtered: {
+            filter: {
+              term: { _id: id }
+            }
+          }
+        }
+      }
+    )
+
+    return nil if result['hits']['total'] == 0
+    result['hits']['hits'].first
+  end
 end
\ No newline at end of file

[gentoo-commits] proj/ag:master commit in: /, lib/

[Alex Legler]

commit:     d3a8004768e26095748003e7932ea500b26eaa6a
Author:     Alex Legler <alex <AT> a3li <DOT> li>
AuthorDate: Mon Feb 23 00:05:20 2015 +0000
Commit:     Alex Legler <a3li <AT> gentoo <DOT> org>
CommitDate: Mon Feb 23 00:05:20 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/ag.git;a=commit;h=d3a80047

Implement --delete

TODO: Actually delete the message

---
 ag             | 8 +++++++-
 lib/storage.rb | 5 +++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/ag b/ag
index 29ee0f2..dbb1584 100755
--- a/ag
+++ b/ag
@@ -142,7 +142,13 @@ def do_incremental
 end
 
 def do_delete
-  abort 'Come back later.'
+  id = Ag::Utils.resolve_id
+
+  begin
+    Ag::Storage.delete($options.name, id)
+  rescue => e
+    $stderr.puts "Cannot delete message: #{e}"
+  end
 end
 
 def do_reindex

diff --git a/lib/storage.rb b/lib/storage.rb
index b4a518e..f255633 100644
--- a/lib/storage.rb
+++ b/lib/storage.rb
@@ -220,6 +220,11 @@ module Ag::Storage
 
     result['hits']['total']
   end
+
+  def delete(list, id)
+    $es.delete(index: 'ml-' + list, type: 'message', id: id)
+  end
+
   def get(list, id)
     result = $es.search(
       index: 'ml-' + list,

[gentoo-commits] proj/ag:master commit in: /, lib/

[Alex Legler]

commit:     88d9781c469e27ca77901a05326f598821715b33
Author:     Alex Legler <alex <AT> a3li <DOT> li>
AuthorDate: Mon Feb 23 20:58:05 2015 +0000
Commit:     Alex Legler <a3li <AT> gentoo <DOT> org>
CommitDate: Mon Feb 23 20:58:05 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/ag.git;a=commit;h=88d9781c

Only occupy 3/4 of the processors

---
 ag             | 6 +++---
 lib/storage.rb | 6 +++---
 lib/utils.rb   | 8 ++++++++
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/ag b/ag
index 73c189b..bcb1ac6 100755
--- a/ag
+++ b/ag
@@ -115,13 +115,13 @@ def do_full
 
   messages = $maildir.list(:cur)
 
-  Parallel.each(messages, progress: "Importing #{$options.name}") do |maildir_message|
+  Parallel.each(messages, in_processes: Ag::Utils.proc_count, progress: "Importing #{$options.name}") do |maildir_message|
     mail = maildir_message.data
 
     begin
       Ag::Storage.store($options.name, mail, maildir_message.unique_name)
     rescue => e
-      $stderr.puts "Cannot save message #{mail.message_id}: #{e.message}"
+      $stderr.puts "Cannot save message #{mail.message_id}: (#{e.class}) #{e.message}" if $options.debug
       next
     end
   end
@@ -132,7 +132,7 @@ end
 def do_incremental
   messages = $maildir.list(:cur)
 
-  Parallel.each(messages, progress: "Importing #{$options.name}") do |maildir_message|
+  Parallel.each(messages, in_processes: Ag::Utils.proc_count, progress: "Importing #{$options.name}") do |maildir_message|
     mail = maildir_message.data
 
     begin

diff --git a/lib/storage.rb b/lib/storage.rb
index 6409df2..5b360f5 100644
--- a/lib/storage.rb
+++ b/lib/storage.rb
@@ -201,9 +201,9 @@ module Ag::Storage
   def fix_threading(list, pass)
     result = $es.search(
       index: 'ml-' + list,
-      size: 100000,
+      size: 5000,
       body: {
-        size: 100000,
+        size: 5000,
         query: {
           filtered: {
             filter: {
@@ -225,7 +225,7 @@ module Ag::Storage
       }
     )
 
-    Parallel.each(result['hits']['hits'], progress: "Calculating Threading (Pass #{pass})") do |hit|
+    Parallel.each(result['hits']['hits'], in_processes: Ag::Utils.proc_count, progress: "Calculating Threading (Pass #{pass})") do |hit|
       msg = resolve_message_id(list, hit['_source']['raw_parent'])
 
       unless msg == nil

diff --git a/lib/utils.rb b/lib/utils.rb
index d621a2e..0213c6d 100644
--- a/lib/utils.rb
+++ b/lib/utils.rb
@@ -1,5 +1,9 @@
 require 'charlock_holmes'
 
+module Parallel::ProcessorCount
+  module_function :processor_count
+end
+
 module Ag
   module Utils
 
@@ -62,5 +66,9 @@ module Ag
     rescue ArgumentError
       ''
     end
+
+    def proc_count
+      (Parallel::ProcessorCount.processor_count.to_f * 0.75).floor
+    end
   end
 end
\ No newline at end of file

[gentoo-commits] proj/ag:master commit in: /, lib/

[Robin H. Johnson]

commit:     45f28da0876f572b0bc5d81b7baeefda1b2b108f
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Mon Feb 23 22:44:14 2015 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Mon Feb 23 22:44:14 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/ag.git;a=commit;h=45f28da0

Include a --jobs parameter.

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

---
 ag           | 7 +++++++
 lib/utils.rb | 9 ++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/ag b/ag
index d36acc4..97c223f 100755
--- a/ag
+++ b/ag
@@ -24,6 +24,7 @@ $options.index_only = false
 $options.no_threading = false
 $options.debug = false
 $options.readonly = false
+$options.jobs = false
 
 op = OptionParser.new do |opts|
   opts.banner = "Usage: ag <<--index-full|--index-new|--delete|--reindex|--info> <--list listname>> <[--file|--msgid|--hash] <maildir/file/hash/messageid>> [options]"
@@ -93,6 +94,10 @@ op = OptionParser.new do |opts|
   opts.on('--readonly', 'Do not alter the maildir in any way') do
     $options.readonly = true
   end
+
+  opts.on('--jobs', 'Number of parallel jobs to run (defaults to 75% of core count)') do |jobs|
+    $options.jobs = jobs
+  end
 end
 op.parse!
 
@@ -110,6 +115,8 @@ $maildir.serializer = Maildir::Serializer::Mail.new
 $es = Elasticsearch::Client.new(log: false)
 $es.transport.reload_connections!
 
+Ag::Utils.proc_count = $options.jobs
+
 ###############################################################################
 
 def do_full

diff --git a/lib/utils.rb b/lib/utils.rb
index 0213c6d..b37437b 100644
--- a/lib/utils.rb
+++ b/lib/utils.rb
@@ -67,8 +67,15 @@ module Ag
       ''
     end
 
+    @proc_count = nil
+    def proc_count=(cnt)
+        cnt = nil if cnt == false
+        @proc_count = cnt
+    end
+
     def proc_count
+      return @proc_count unless @proc_count.nil? or @proc_count == false
       (Parallel::ProcessorCount.processor_count.to_f * 0.75).floor
     end
   end
-end
\ No newline at end of file
+end

[gentoo-commits] proj/ag:master commit in: /, lib/

[Robin H. Johnson]

commit:     149ce341820a66010e8592975d86e5d6dd0c6b7e
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 24 01:47:30 2015 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Tue Feb 24 01:49:07 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/ag.git;a=commit;h=149ce341

Expose delete index.

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

---
 ag             | 23 +++++++++++++++++++----
 lib/storage.rb | 11 +++++++++--
 2 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/ag b/ag
index c5056ff..4636675 100755
--- a/ag
+++ b/ag
@@ -27,7 +27,7 @@ $options.readonly = false
 $options.jobs = false
 
 op = OptionParser.new do |opts|
-  opts.banner = "Usage: ag <<--index-full|--index-new|--delete|--reindex|--info> <--list listname>> <[--file|--msgid|--hash] <maildir/file/hash/messageid>> [options]"
+  opts.banner = "Usage: ag <<--index-full|--index-new|--delete-msg|--delete-index|--reindex|--info> <--list listname>> <[--file|--msgid|--hash] <maildir/file/hash/messageid>> [options]"
 
   opts.on('--index-full', 'Read the full past archive from the .cur Maildir') do
     abort 'Can only select one action' if $options.action != nil
@@ -41,10 +41,16 @@ op = OptionParser.new do |opts|
     $options.action = :do_incremental
   end
 
-  opts.on('--delete', 'Delete message. Needs --file, --msgid, or --hash') do
+  opts.on('--delete-msg', 'Delete message. Needs --file, --msgid, or --hash') do
     abort 'Can only select one action' if $options.action != nil
 
-    $options.action = :do_delete
+    $options.action = :do_delete_msg
+  end
+  
+  opts.on('--delete-Index', 'Delete index. Needs --list') do
+    abort 'Can only select one action' if $options.action != nil
+
+    $options.action = :do_index
   end
 
   opts.on('--info', 'Display message details. Needs --file, --msgid, or --hash') do
@@ -156,7 +162,7 @@ def do_incremental
   Ag::Threading.calc($options.name) unless $options.no_threading
 end
 
-def do_delete
+def do_delete_msg
   id = Ag::Utils.resolve_id
 
   begin
@@ -166,6 +172,15 @@ def do_delete
   end
 end
 
+def do_delete_index
+  begin
+    Ag::Storage.delete_index($options.name)
+  rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
+  rescue => e
+    $stderr.puts "Cannot delete index: #{e}"
+  end
+end
+
 def do_reindex
   abort 'Come back later.'
 end

diff --git a/lib/storage.rb b/lib/storage.rb
index 5b360f5..656a6bf 100644
--- a/lib/storage.rb
+++ b/lib/storage.rb
@@ -3,9 +3,16 @@ require 'date'
 
 module Ag::Storage
   module_function
+
+  # Throws Elasticsearch::Transport::Transport::Errors::NotFound
+  # if the list does not exist
+  def delete_index(list)
+      $es.indices.delete index: 'ml-' + list
+  end
+
   def create_index(list)
     begin
-      $es.indices.delete index: 'ml-' + list
+      delete_index(ist)
     rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
       $stderr.puts "Index did not exist yet. Creating." if $options.debug
     end
@@ -267,4 +274,4 @@ module Ag::Storage
     return nil if result['hits']['total'] == 0
     result['hits']['hits'].first
   end
-end
\ No newline at end of file
+end

[gentoo-commits] proj/ag:master commit in: /, lib/

[Robin H. Johnson]

commit:     ef43e934de266b49dec9373cb6c281c0ee7c67dc
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 24 02:16:55 2015 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Tue Feb 24 02:16:55 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/ag.git;a=commit;h=ef43e934

Improve delete code paths.

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

---
 ag             | 37 +++++++++++++++++++++++--------------
 lib/storage.rb |  4 ++++
 lib/utils.rb   |  6 ++++++
 3 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/ag b/ag
index 57dc0a4..ffe714b 100755
--- a/ag
+++ b/ag
@@ -26,41 +26,46 @@ $options.debug = false
 $options.readonly = false
 $options.jobs = false
 $options.progress = true
+$options.need_argument = true
+$options.argmode = nil
 
 op = OptionParser.new do |opts|
   opts.banner = "Usage: ag <<--index-full|--index-new|--delete-msg|--delete-index|--reindex|--info> <--list listname>> <[--file|--msgid|--hash] <maildir/file/hash/messageid>> [options]"
 
-  opts.on('--index-full', 'Read the full past archive from the .cur Maildir') do
+  opts.on('--index-full', 'Read the full past archive from Maildir/cur. Needs --list and a Maildir') do
     abort 'Can only select one action' if $options.action != nil
 
     $options.action = :do_full
+    $options.argmode = :dir
   end
 
-  opts.on('--index-new', 'Read new messages from .new and move them to .cur') do
+  opts.on('--index-new', 'Read new messages from Maildir/new and move them to Maildir/cur. Needs --list and a Maildir') do
     abort 'Can only select one action' if $options.action != nil
 
     $options.action = :do_incremental
+    $options.argmode = :dir
   end
 
-  opts.on('--delete-msg', 'Delete message. Needs --file, --msgid, or --hash') do
+  opts.on('--delete-msg', 'Delete message. Needs --list and one of --file, --msgid, or --hash') do
     abort 'Can only select one action' if $options.action != nil
 
     $options.action = :do_delete_msg
   end
-  
-  opts.on('--delete-Index', 'Delete index. Needs --list') do
+
+  opts.on('--delete-index', 'Delete index. Needs --list') do
     abort 'Can only select one action' if $options.action != nil
 
     $options.action = :do_index
+    $options.need_argument = false
   end
 
-  opts.on('--info', 'Display message details. Needs --file, --msgid, or --hash') do
+  opts.on('--info', 'Display message details. Needs --list and one of --file, --msgid, or --hash') do
     abort 'Can only select one action' if $options.action != nil
 
     $options.action = :do_info
   end
 
-  opts.on('--reindex', 'Reindex message. Needs --file') do
+  opts.on('--reindex', 'Reindex message. Needs --list and --file') do
     abort 'Can only select one action' if $options.action != nil
 
     $options.action = :do_reindex
@@ -105,7 +110,7 @@ op = OptionParser.new do |opts|
   opts.on('--jobs JOBS', 'Number of parallel jobs to run (defaults to 75% of core count)') do |jobs|
     $options.jobs = jobs.to_i
   end
-  
+
   opts.on('--progress', 'Display the progress bar') do
     $options.progress = true
   end
@@ -117,11 +122,13 @@ op.parse!
 
 abort op.help unless $options.action
 abort 'List name required' unless $options.name
-$options.dir = ARGV[0] or abort 'Need a Maildir/File/Hash/Message-Id to work with'
+$options.dir = ARGV[0] or abort 'Need a Maildir/File/Hash/Message-Id to work with' if $options.need_argument
 
-# Open maildir and set serializer
-$maildir = Maildir.new(File.join($options.dir), false)
-$maildir.serializer = Maildir::Serializer::Mail.new
+if($options.argmode == :dir)
+  # Open maildir and set serializer
+  $maildir = Maildir.new(File.join($options.dir), false)
+  $maildir.serializer = Maildir::Serializer::Mail.new
+end
 
 # Connect to Elasticsearch
 $es = Elasticsearch::Client.new(log: false)
@@ -132,12 +139,13 @@ Ag::Utils.proc_count = $options.jobs
 ###############################################################################
 
 def do_full
+  abort "Wrong argument type: #{$options.argmode.to_s}" unless $options.argmode == :dir
   Ag::Storage.create_index($options.name)
 
   messages = $maildir.list(:cur)
 
   opts = {
-	  :in_processes => Ag::Utils.proc_count,
+    :in_processes => Ag::Utils.proc_count,
   }
   opts[:progress] = "Importing #{$options.name}" if $options.progress
   Parallel.each(messages, opts) do |maildir_message|
@@ -155,10 +163,11 @@ def do_full
 end
 
 def do_incremental
+  abort "Wrong argument type: #{$options.argmode.to_s}" unless $options.argmode == :dir
   messages = $maildir.list(:cur)
 
   opts = {
-	  :in_processes => Ag::Utils.proc_count,
+    :in_processes => Ag::Utils.proc_count,
   }
   opts[:progress] = "Importing #{$options.name}" if $options.progress
   Parallel.each(messages, opts) do |maildir_message|

diff --git a/lib/storage.rb b/lib/storage.rb
index 8ab2d4e..68de268 100644
--- a/lib/storage.rb
+++ b/lib/storage.rb
@@ -141,6 +141,10 @@ module Ag::Storage
     resolve_by_field(list, :raw_filename, filename)
   end
 
+  def resolve_hash(list, hash = nil)
+    resolve_by_field(list, :_id, hash)
+  end
+
   def store(list, message, filename)
     content = get_content(message, filename)
 

diff --git a/lib/utils.rb b/lib/utils.rb
index 38349e0..b9156cd 100644
--- a/lib/utils.rb
+++ b/lib/utils.rb
@@ -33,6 +33,12 @@ module Ag
         id = Ag::Storage.resolve_message_id($options.name, $options.dir)
       when :file
         id = Ag::Storage.resolve_filename($options.name, $options.dir)
+      when :hash
+        id = Ag::Storage.resolve_hash($options.name, $options.dir)
+      when :dir
+        abort 'Cannot perform resolve a directory to a message id'
+      when nil
+        abort 'Cannot resolve a nil id'
       end
 
       id