1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
use std::collections::HashMap;
use std::error::Error;
use std::fs::File;
use std::io;
use std::io::ErrorKind;
use std::io::Write;
use std::ops::Deref;
use std::path::Path;
use std::sync::Arc;
use std::sync::Mutex;

use time;

use crate::dispatcher::server;
use crate::helpers;
use crate::helpers::{TaskProgress, TaskReport, TaskStatus};
use crate::models::{Service, WorkerMetadata};

/// Specifies the binding and operation parameters for a ZMQ sink component
pub struct Sink {
  /// port to listen on
  pub port: usize,
  /// the size of the dispatch queue
  /// (also the batch size for Task store queue requests)
  pub queue_size: usize,
  /// size of an individual message chunk sent via zeromq
  /// (keep this small to avoid large RAM use, increase to reduce network bandwidth)
  pub message_size: usize,
  /// address for the Task store postgres endpoint
  pub backend_address: String,
}

impl Sink {
  /// Starts a receiver/sink `Server` (ZMQ Pull), to accept processing responses.
  /// The sink shares state with other manager threads via queues for tasks in progress,
  /// as well as a queue for completed tasks pending persisting to disk.
  /// A job limit can be provided as a termination condition for the sink server.
  pub fn start(
    &self,
    services_arc: &Arc<Mutex<HashMap<String, Option<Service>>>>,
    progress_queue_arc: &Arc<Mutex<HashMap<i64, TaskProgress>>>,
    done_queue_arc: &Arc<Mutex<Vec<TaskReport>>>,
    job_limit: Option<usize>,
  ) -> Result<(), Box<dyn Error>>
  {
    // Ok, let's bind to a port and start broadcasting
    let context = zmq::Context::new();
    let sink = context.socket(zmq::PULL)?;
    let address = format!("tcp://*:{}", self.port);
    assert!(sink.bind(&address).is_ok());

    let mut sink_job_count: usize = 0;

    loop {
      let mut recv_msg = zmq::Message::new();
      let mut identity_msg = zmq::Message::new();
      let mut taskid_msg = zmq::Message::new();
      let mut service_msg = zmq::Message::new();

      sink.recv(&mut identity_msg, 0)?;
      let identity = match identity_msg.as_str() {
        Some(some_name) => some_name,
        None => "_worker_",
      };

      sink.recv(&mut service_msg, 0)?;
      let service_name = match service_msg.as_str() {
        Some(some_name) => some_name,
        None => "_unknown_",
      };

      sink.recv(&mut taskid_msg, 0)?;
      let taskid_str = match taskid_msg.as_str() {
        Some(some_id) => some_id,
        None => "-1",
      };
      let taskid = match taskid_str.parse::<i64>() {
        Ok(some_id) => some_id,
        Err(_) => -1,
      };

      // We have a job, count it
      sink_job_count += 1;
      let mut total_incoming = 0;
      let request_time = time::get_time();
      println!(
        "sink {:?}: incoming result for {:?}, worker {:?}, taskid: {}",
        sink_job_count, service_name, identity, taskid
      );

      if let Some(task_progress) = server::pop_progress_task(&progress_queue_arc, taskid) {
        let task = task_progress.task;
        match server::get_service(service_name, &services_arc) {
          None => {
            return Err(Box::new(io::Error::new(
              ErrorKind::Other,
              "TODO: Server::get_service found nothing.",
            )));
          }, // TODO: Handle errors
          Some(service) => {
            if service.id == task.service_id {
              // println!("Task and Service match up.");
              if service.id == 1 {
                // No payload needed for init
                sink.recv(&mut recv_msg, 0)?;
                let done_report = TaskReport {
                  task: task.clone(),
                  status: TaskStatus::NoProblem,
                  messages: Vec::new(),
                };
                server::push_done_queue(&done_queue_arc, done_report);
              } else {
                // Receive the rest of the input in the correct file
                match Path::new(&task.entry.clone()).parent() {
                  None => {
                    println!("-- Error TODO: Path::new(&task.entry).parent() failed.");
                  },
                  Some(recv_dir) => {
                    match recv_dir.to_str() {
                      None => {
                        println!("-- Error TODO: recv_dir.to_str() failed");
                      },
                      Some(recv_dir_str) => {
                        let recv_dir_string = recv_dir_str.to_string();
                        let recv_pathname = recv_dir_string + "/" + &service.name + ".zip";
                        let recv_path = Path::new(&recv_pathname);
                        // println!("Will write to {:?}", recv_path);
                        {
                          // Explicitly scope file, so that we drop it the moment we are done
                          // writing.
                          let mut file = match File::create(recv_path) {
                            Ok(f) => f,
                            Err(e) => {
                              println!("-- Error TODO: File::create(recv_path): {:?}", e);
                              continue;
                            },
                          };
                          while let Ok(_) = sink.recv(&mut recv_msg, 0) {
                            match file.write(recv_msg.deref()) {
                              Ok(written_bytes) => total_incoming += written_bytes,
                              Err(e) => {
                                println!(
                                  "-- Error TODO: file.write(recv_msg.deref()) failed: {:?}",
                                  e
                                );
                                break;
                              },
                            };
                            match sink.get_rcvmore() {
                              Ok(true) => {}, // keep receiving
                              _ => break,     /* println!("Error TODO: sink.get_rcvmore failed:
                                                * {:?}", e); */
                            };
                          }
                          drop(file);
                        }
                        // Then mark the task done. This can be in a new thread later on
                        let done_report = helpers::generate_report(task, recv_path);
                        server::push_done_queue(&done_queue_arc, done_report);
                      },
                    }
                  },
                }
              }
              // Also update worker metadata for transparency
              WorkerMetadata::record_received(
                identity.to_string(),
                service.id,
                taskid,
                self.backend_address.clone(),
              )?;
            } else {
              // Otherwise just discard the rest of the message
              println!(
                "-- Mismatch between requested service id {:?} and task's service id {:?} for task {:?}, discarding response",
                service.id, task.service_id, taskid
              );
              while let Ok(_) = sink.recv(&mut recv_msg, 0) {
                if !sink.get_rcvmore()? {
                  break;
                }
              }
            }
          },
        };
      } else {
        // No such task, just discard the next message from the sink
        println!("-- No such task id found in dispatcher queue: {:?}", taskid);
        while let Ok(_) = sink.recv(&mut recv_msg, 0) {
          if !sink.get_rcvmore()? {
            break;
          }
        }
      }
      let responded_time = time::get_time();
      let request_duration = (responded_time - request_time).num_milliseconds();
      println!(
        "sink {}: message size: {}, took {}ms.",
        sink_job_count, total_incoming, request_duration
      );
      if let Some(limit_number) = job_limit {
        if sink_job_count >= limit_number {
          println!(
            "sink {}: job limit reached, terminating Sink thread...",
            limit_number
          );
          break;
        }
      }
    }
    Ok(())
  }
}