#include <config.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <stdlib.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
#include <syslog.h>
#include <string.h>
#include "annotate.h"
#include "assert.h"
#include "mboxlist.h"
#include "global.h"
#include "exitcodes.h"
#include "imap_err.h"
#include "mailbox.h"
#include "xmalloc.h"
#include "acl.h"
#include "seen.h"
#include "mboxname.h"
#include "map.h"
#include "squat.h"
#include "imapd.h"
const int config_need_data = CONFIG_NEED_PARTITION_DATA;
extern char *optarg;
extern int optind;
static struct namespace squat_namespace;
int imapd_exists;
struct protstream *imapd_out = NULL;
struct auth_state *imapd_authstate = NULL;
char *imapd_userid = NULL;
void printastring(const char *s __attribute__((unused)))
{
fatal("not implemented", EC_SOFTWARE);
}
typedef struct {
int indexed_bytes;
int indexed_messages;
int index_size;
time_t start_time;
time_t end_time;
} SquatStats;
const int SKIP_FUZZ = 60;
static int verbose = 0;
static int mailbox_count = 0;
static int skip_unmodified = 0;
static SquatStats total_stats;
static void start_stats(SquatStats* stats) {
stats->index_size = 0;
stats->indexed_bytes = 0;
stats->indexed_messages = 0;
stats->start_time = time(NULL);
}
static void stop_stats(SquatStats* stats) {
stats->end_time = time(NULL);
}
static void print_stats(FILE* out, SquatStats* stats) {
fprintf(out, "Indexed %d messages (%d bytes) "
"into %d index bytes in %d seconds\n",
stats->indexed_messages, stats->indexed_bytes,
stats->index_size, (int) (stats->end_time - stats->start_time));
}
static int usage(const char *name)
{
fprintf(stderr,
"usage: %s [-C <alt_config>] [-r] [-s] [-a] [-v] [mailbox...]\n",
name);
exit(EC_USAGE);
}
static void fatal_syserror(const char* s)
{
perror(s);
exit(99);
}
static void fatal_squat_error(const char* s)
{
int err = squat_get_last_error();
switch (err) {
case SQUAT_ERR_OUT_OF_MEMORY:
fprintf(stderr, "SQUAT: Out of memory (%s)\n", s);
break;
case SQUAT_ERR_SYSERR:
perror(s);
break;
default:
fprintf(stderr, "SQUAT: Unknown error %d (%s)\n", err, s);
}
exit(98);
}
typedef struct {
SquatStats* mailbox_stats;
SquatIndex* index;
struct mailbox* mailbox;
} SquatReceiverData;
static void search_text_receiver(int uid, int part, int cmd,
char const* text, int text_len, void* rock) {
SquatReceiverData* d = (SquatReceiverData*)rock;
if ((cmd & SEARCHINDEX_CMD_BEGINPART) != 0) {
char buf[100];
char part_char;
switch (part) {
case SEARCHINDEX_PART_FROM: part_char = 'f'; break;
case SEARCHINDEX_PART_TO: part_char = 't'; break;
case SEARCHINDEX_PART_CC: part_char = 'c'; break;
case SEARCHINDEX_PART_BCC: part_char = 'b'; break;
case SEARCHINDEX_PART_SUBJECT: part_char = 's'; break;
case SEARCHINDEX_PART_HEADERS: part_char = 'h'; break;
default:
assert(0);
case SEARCHINDEX_PART_BODY:
part_char = 'm';
d->mailbox_stats->indexed_messages++;
total_stats.indexed_messages++;
break;
}
snprintf(buf, sizeof(buf), "%c%d", part_char, uid);
if ((cmd & SEARCHINDEX_CMD_ENDPART) != 0
&& ((cmd & SEARCHINDEX_CMD_APPENDPART) == 0
|| text_len < SQUAT_WORD_SIZE)) {
if (verbose > 2) {
printf("Skipping tiny document part '%s' (size %d)\n", buf,
(cmd & SEARCHINDEX_CMD_APPENDPART) == 0 ? 0 : text_len);
}
return;
}
if (verbose > 2) {
printf("Opening document part '%s'\n", buf);
}
if (squat_index_open_document(d->index, buf) != SQUAT_OK) {
fatal_squat_error("Writing index");
}
}
if ((cmd & SEARCHINDEX_CMD_APPENDPART) != 0) {
if (verbose > 3) {
printf("Writing %d bytes into message %d\n", text_len, uid);
}
if (squat_index_append_document(d->index, text, text_len) != SQUAT_OK) {
fatal_squat_error("Writing index data");
}
d->mailbox_stats->indexed_bytes += text_len;
total_stats.indexed_bytes += text_len;
}
if ((cmd & SEARCHINDEX_CMD_ENDPART) != 0) {
if (squat_index_close_document(d->index) != SQUAT_OK) {
fatal_squat_error("Writing index update");
}
}
}
static void stats_callback(void* closure __attribute__((unused)),
SquatStatsEvent* params) {
switch (params->generic.type) {
case SQUAT_STATS_COMPLETED_INITIAL_CHAR:
if (verbose > 1) {
if (params->completed_initial_char.num_words > 0) {
printf("Processing index character %d, %d total words, "
"temp file size is %d\n",
params->completed_initial_char.completed_char,
params->completed_initial_char.num_words,
params->completed_initial_char.temp_file_size);
}
}
break;
default:
;
}
}
static int index_me(char *name, int matchlen __attribute__((unused)),
int maycreate __attribute__((unused)),
void *rock) {
struct mailbox m;
int r;
SquatStats stats;
SquatReceiverData data;
char tmp_file_name[1000];
char index_file_name[1000];
int fd;
SquatOptions options;
struct stat index_file_info;
struct stat tmp_file_info;
char uid_validity_buf[30];
char extname[MAX_MAILBOX_NAME+1];
int use_annot = *((int *) rock);
if (use_annot) {
char buf[MAX_MAILBOX_NAME+1] = "", *p;
struct annotation_data attrib;
int domainlen = 0;
if (config_virtdomains && (p = strchr(name, '!')))
domainlen = p - name + 1;
strlcpy(buf, name, sizeof(buf));
while (1) {
r = annotatemore_lookup(buf, "/vendor/cmu/cyrus-imapd/squat", "",
&attrib);
if (r ||
attrib.value ||
!buf[0]) {
break;
}
p = strrchr(buf, '.');
if (p && (p - buf > domainlen))
*p = '\0';
else if (!buf[domainlen])
buf[0] = '\0';
else
buf[domainlen] = '\0';
}
if (r || !attrib.value || strcasecmp(attrib.value, "true"))
return 0;
}
(*squat_namespace.mboxname_toexternal)(&squat_namespace, name,
NULL, extname);
data.mailbox_stats = &stats;
data.mailbox = &m;
memset(&m, 0, sizeof(struct mailbox));
r = mailbox_open_header(name, 0, &m);
if (r) {
if (verbose) {
printf("error opening %s: %s\n", extname, error_message(r));
}
return 1;
}
r = mailbox_open_index(&m);
if (!r) r = mailbox_lock_pop(&m);
if (r) {
if (verbose) {
printf("error locking index %s: %s\n", extname, error_message(r));
}
mailbox_close(&m);
return 1;
}
snprintf(index_file_name, sizeof(index_file_name),
"%s%s", m.path, FNAME_SQUAT_INDEX);
snprintf(tmp_file_name, sizeof(tmp_file_name),
"%s%s", m.path, FNAME_INDEX);
if (skip_unmodified &&
!stat(tmp_file_name, &tmp_file_info) &&
!stat(index_file_name, &index_file_info)) {
if (SKIP_FUZZ + tmp_file_info.st_mtime <
index_file_info.st_mtime) {
syslog(LOG_DEBUG, "skipping mailbox %s", extname);
if (verbose > 0) {
printf("Skipping mailbox %s\n", extname);
}
mailbox_close(&m);
return 0;
}
}
snprintf(tmp_file_name, sizeof(tmp_file_name),
"%s%s.tmp", m.path, FNAME_SQUAT_INDEX);
syslog(LOG_INFO, "indexing mailbox %s... ", extname);
if (verbose > 0) {
printf("Indexing mailbox %s... ", extname);
}
if ((fd = open(tmp_file_name,
O_CREAT | O_TRUNC | O_WRONLY, S_IREAD | S_IWRITE))
< 0) {
fatal_syserror("Unable to create temporary index file");
}
options.option_mask = SQUAT_OPTION_TMP_PATH | SQUAT_OPTION_STATISTICS;
options.tmp_path = m.path;
options.stats_callback = stats_callback;
options.stats_callback_closure = NULL;
data.index = squat_index_init(fd, &options);
if (data.index == NULL) {
fatal_squat_error("Initializing index");
}
snprintf(uid_validity_buf, sizeof(uid_validity_buf),
"validity.%ld", m.uidvalidity);
if (squat_index_open_document(data.index, uid_validity_buf) != SQUAT_OK
|| squat_index_close_document(data.index) != SQUAT_OK) {
fatal_squat_error("Writing index");
}
start_stats(&stats);
mailbox_read_index_header(&m);
index_operatemailbox(&m);
index_getsearchtext(&m, search_text_receiver, &data);
index_closemailbox(&m);
mailbox_close(&m);
mailbox_count++;
if (squat_index_finish(data.index) != SQUAT_OK) {
fatal_squat_error("Closing index");
}
if (fstat(fd, &index_file_info) < 0) {
fatal_syserror("Unable to stat temporary index file");
}
stats.index_size = index_file_info.st_size;
total_stats.index_size += index_file_info.st_size;
if (close(fd) < 0) {
fatal_syserror("Unable to complete writing temporary index file");
}
if (rename(tmp_file_name, index_file_name) < 0) {
fatal_syserror("Unable to rename temporary index file");
}
stop_stats(&stats);
if (verbose > 0) {
print_stats(stdout, &stats);
}
return 0;
}
int main(int argc, char **argv)
{
int opt;
char *alt_config = NULL;
int rflag = 0, use_annot = 0;
int i;
char buf[MAX_MAILBOX_PATH+1];
int r;
if(geteuid() == 0)
fatal("must run as the Cyrus user", EC_USAGE);
setbuf(stdout, NULL);
while ((opt = getopt(argc, argv, "C:rsav")) != EOF) {
switch (opt) {
case 'C':
alt_config = optarg;
break;
case 'v':
verbose++;
break;
case 'r':
rflag = 1;
break;
case 's':
skip_unmodified = 1;
break;
case 'a':
use_annot = 1;
break;
default:
usage("squatter");
}
}
cyrus_init(alt_config, "squatter", 0);
syslog(LOG_NOTICE, "indexing mailboxes");
if ((r = mboxname_init_namespace(&squat_namespace, 1)) != 0) {
fatal(error_message(r), EC_CONFIG);
}
annotatemore_init(0, NULL, NULL);
annotatemore_open(NULL);
mboxlist_init(0);
mboxlist_open(NULL);
mailbox_initialize();
start_stats(&total_stats);
if (optind == argc) {
if (rflag) {
fprintf(stderr, "please specify a mailbox to recurse from\n");
exit(EC_USAGE);
}
assert(!rflag);
strlcpy(buf, "*", sizeof(buf));
(*squat_namespace.mboxlist_findall)(&squat_namespace, buf, 1,
0, 0, index_me, &use_annot);
}
for (i = optind; i < argc; i++) {
(*squat_namespace.mboxname_tointernal)(&squat_namespace, argv[i],
NULL, buf);
index_me(buf, 0, 0, &use_annot);
if (rflag) {
strlcat(buf, ".*", sizeof(buf));
(*squat_namespace.mboxlist_findall)(&squat_namespace, buf, 1,
0, 0, index_me, &use_annot);
}
}
if (verbose > 0 && mailbox_count > 1) {
stop_stats(&total_stats);
printf("Total over all mailboxes: ");
print_stats(stdout, &total_stats);
}
syslog(LOG_NOTICE, "done indexing mailboxes");
seen_done();
mboxlist_close();
mboxlist_done();
annotatemore_close();
annotatemore_done();
cyrus_done();
return 0;
}