Failed to save partial JSON Element 15382518

def save_partial_json_to_file(all_data: false, operation: '(updated)', changes: nil, timestamp: nil)
  if $FORWARD_DATA_CHANGES != true
    Cupix::Logger.debug('Data changes forwarding is disabled', class: self.class, module: 'DataWareHouse', function: 'save_partial_json_to_file')
    return nil
  end

  generate_and_save_partial_json(operation, all_data: all_data, changes: changes || saved_changes, timestamp: timestamp)
end

def generate_and_save_partial_json(operation, all_data: false, changes: nil, timestamp: nil)
  json_data = prepare_partial_json_data(operation, all_data: all_data, changes: changes)
  json_content = ActiveSupport::Gzip.compress(json_data.to_json)
  save_partial_json_file(json_content, partial_json_object_key(timestamp: timestamp))
end

def save_partial_json_file(content, path)
  filepath = Rails.env.development? ? "./tmp/storage/#{path}" : "/var/data_changes/#{path}"
  create_directory_if_not_exists(filepath)

  # Verify directory is writable before attempting file write
  dirname = File.dirname(filepath)
  unless File.writable?(dirname)
    dir_stat = File.stat(dirname)
    Cupix::Logger.error(
      'Directory not writable before file write attempt',
      class: self.class.name, module: 'DataWareHouse', function: 'save_partial_json_file',
      directory: dirname, permissions: format('%o', dir_stat.mode & 0o777),
      uid: dir_stat.uid, gid: dir_stat.gid, process_uid: Process.uid, process_euid: Process.euid
    )
    raise Errno::EACCES, "Directory #{dirname} is not writable"
  end

  File.open(filepath, 'wb') { |file| file.write(content) }
rescue Errno::EACCES, Errno::ENOSPC => e
  Cupix::Logger.error("Failed to save Partial JSON #{self.class.name} #{self.id} to file #{filepath}",
    class: self.class.name, module: 'DataWareHouse', function: 'save_partial_json_file',
    error: e.message, backtrace: e.backtrace)
else
  Cupix::Logger.debug("Successfully saved Partial JSON #{self.class.name} #{self.id} to file #{filepath}",
    class: self.class.name, module: 'DataWareHouse', function: 'save_partial_json_file')
end

models.find_in_batches(batch_size: 50) do |groups|
  groups.each do |model|
    model.save_partial_json_to_file(all_data: true, operation: operation, timestamp: timestamp)
  end
end
rescue NoMethodError => e
  Cupix::Logger.error(e.message, class: self.class, function: 'perform')
end

service:cupixworks-worker status:error @environment:production "Failed to save partial JSON"

service:cupixworks-worker status:error @environment:production "Failed to save Partial JSON" @function:save_partial_json_file

{
  "timestamp": "2026-05-22T20:20:42.030Z",
  "message": "Failed to save partial JSON Element 15382518",
  "host": "ip-10-1-18-233.us-west-2.compute.internal",
  "class": "Element",
  "module": "DataWareHouse",
  "function": "bulk_partial_save_to_file",
  "error": "No space left on device @ rb_sysopen - /var/data_changes/elements/partial/2026-05-22/uswe2/1779481236606406-elements-15382518.json.gz"
}

{
  "timestamp": "2026-05-22T23:50:00Z",
  "message": "Failed to save Partial JSON Record 129985 to file /var/data_changes/records/partial/2026-05-22/uswe2/1779494399564283-records-129985.json.gz",
  "host": "ip-10-1-18-233.us-west-2.compute.internal",
  "class": "Record",
  "module": "DataWareHouse",
  "function": "save_partial_json_file",
  "error": "No space left on device @ rb_sysopen"
}

{
  "timestamp": "2026-05-22T23:41:00Z",
  "message": "ES bulk partial index dead",
  "class": "BulkPartialIndexDeadWorker",
  "function": "perform",
  "reason": "doc_missing_after_max_retries",
  "model": "ElementTrace"
}

service:cupixworks-worker status:error "No space left on device"

service:cupixworks-worker status:error @function:save_partial_json_file