refactoring, README improved, env vars changed, v0.1.1

Author: karolselak

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    travis-backup (0.1.0)
+    travis-backup (0.1.1)
       activerecord
       bootsnap
       pg

README.md

@@ -1,6 +1,6 @@
 # README
 
-*travis-backup* is an application that helps with housekeeping and backup for Travis CI database v2.2 and with migration to v3.0 database.
+*travis-backup* is an application that helps with housekeeping and backup for Travis CI database v2.2 and with migration to v3.0 database. By default it removes requests and builds with their corresponding jobs and logs, as long as they are older than given threshold says (and backups them in files, if this option is active). Although it can be also run with special modes: `move_logs`, for moving logs from one database to another, and `remove_orphans`, for deleting all orphaned data.
 
 ### Installation and run
 
@@ -75,15 +75,19 @@ backup:
   limit: 1000               # builds limit for one backup file
   threshold: 6              # number of months from now - data younger than this time won't be backuped
   files_location: './dump'  # path of the folder in which backup files will be placed
-  user_id                   # run only for given user
-  org_id                    # run only for given organization
-  repo_id                   # run only for given repository
+  user_id: 1                # run only for given user
+  org_id: 1                 # run only for given organization
+  repo_id: 1                # run only for given repository
+  move_logs: false          # run in move logs mode - move all logs to database at destination_db_url URL
+  remove_orphans: false     # run in remove orphans mode
 ```
 
-You can also set these properties using env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `USER_ID`, `ORG_ID`, `REPO_ID`.
+You can also set these properties using env vars corresponding to them: `IF_BACKUP`, `BACKUP_DRY_RUN`, `BACKUP_LIMIT`, `BACKUP_THRESHOLD`, `BACKUP_FILES_LOCATION`, `BACKUP_USER_ID`, `BACKUP_ORG_ID`, `BACKUP_REPO_ID`, `BACKUP_MOVE_LOGS`, `BACKUP_REMOVE_ORPHANS`.
 
 You should also specify your database url. You can do this the standard way in `config/database.yml` file, setting the `database_url` hash argument while creating `Backup` instance or using the `DATABASE_URL` env var. Your database should be consistent with the Travis 2.2 database schema.
 
+For `move_logs` mode you need also to specify a destination database. You can set it also in `config/database.yml` file, in `destination` subsection, setting the `destination_db_url` hash argument while creating `Backup` instance or using the `BACKUP_DESTINATION_DB_URL` env var. Your destination database should be consistent with the Travis 3.0 database schema.
+
 ### How to run the test suite
 
 You can run the test after cloning this repository. Next you should call

lib/config.rb

@@ -67,19 +67,19 @@ def set_values(args)
     @user_id = first_not_nil(
       args[:user_id],
       argv_opts[:user_id],
-      ENV['USER_ID'],
+      ENV['BACKUP_USER_ID'],
       config.dig('backup', 'user_id')
     )
     @repo_id = first_not_nil(
       args[:repo_id],
       argv_opts[:repo_id],
-      ENV['REPO_ID'],
+      ENV['BACKUP_REPO_ID'],
       config.dig('backup', 'repo_id')
     )
     @org_id = first_not_nil(
       args[:org_id],
       argv_opts[:org_id],
-      ENV['ORG_ID'],
+      ENV['BACKUP_ORG_ID'],
       config.dig('backup', 'org_id')
     )
     @move_logs = first_not_nil(

lib/travis-backup.rb

@@ -51,21 +51,32 @@ def run(args={})
     elsif @config.remove_orphans
       remove_orphans
     elsif owner_id
-      Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
-        process_repo(repository)
-      end
+      process_repos_for_owner(owner_id, owner_type)
     elsif repo_id
-      repository = Repository.find(repo_id)
-      process_repo(repository)
+      process_repo_with_id(repo_id)
     else
-      Repository.order(:id).each do |repository|
-        process_repo(repository)
-      end
+      process_all_repos
     end
 
     print_dry_run_report if @config.dry_run
   end
 
+  def process_repos_for_owner(owner_id, owner_type)
+    Repository.where('owner_id = ? and owner_type = ?', owner_id, owner_type).order(:id).each do |repository|
+      process_repo(repository)
+    end
+  end
+
+  def process_repo_with_id(repo_id)
+    process_repo(Repository.find(repo_id))
+  end
+
+  def process_all_repos
+    Repository.order(:id).each do |repository|
+      process_repo(repository)
+    end
+  end
+
   def print_dry_run_report
     if @dry_run_report.to_a.map(&:second).flatten.empty?
       puts 'Dry run active. No data would be removed in normal run.'
@@ -168,23 +179,15 @@ def remove_orphans_for_table(model_class, table_a_name, table_b_name, fk_name)
     })
 
     if config.dry_run
-      dry_run_report[table_a_name.to_sym] = [] if dry_run_report[table_a_name.to_sym].nil?
-      dry_run_report[table_a_name.to_sym].concat(for_delete.map(&:id))
-      dry_run_report[table_a_name.to_sym].uniq!
+      key = table_a_name.to_sym
+      dry_run_report[key] = [] if dry_run_report[key].nil?
+      dry_run_report[key].concat(for_delete.map(&:id))
+      dry_run_report[key].uniq!
     else
       model_class.where(id: for_delete.map(&:id)).delete_all
     end
   end
 
-  def process_repo_builds(repository) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-    threshold = @config.threshold.to_i.months.ago.to_datetime
-    current_build_id = repository.current_build_id || -1
-    repository.builds.where('created_at < ? and id != ?', threshold, current_build_id)
-              .in_groups_of(@config.limit.to_i, false).map do |builds_batch|
-      @config.if_backup ? save_and_destroy_builds_batch(builds_batch, repository) : destroy_builds_batch(builds_batch)
-    end.compact
-  end
-
   private
 
   def save_and_destroy_builds_batch(builds_batch, repository)

travis-backup.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name                  = 'travis-backup'
-  s.version               = '0.1.0'
+  s.version               = '0.1.1'
   s.summary               = 'Travis CI backup tool'
   s.authors               = ['Karol Selak']
   s.required_ruby_version = Gem::Requirement.new(">= 2.3.0")