svn commit: r447975 - in head/mail/dovecot: . files
Adam Weinberger
adamw at FreeBSD.org
Tue Aug 15 01:49:02 UTC 2017
Author: adamw
Date: Tue Aug 15 01:49:01 2017
New Revision: 447975
URL: https://svnweb.freebsd.org/changeset/ports/447975
Log:
Apply upstream patches for indexing.
Reviewed by: ler
Obtained from: dovecot GH
Added:
head/mail/dovecot/files/patch-UPSTREAM-indexing (contents, props changed)
Modified:
head/mail/dovecot/Makefile
Modified: head/mail/dovecot/Makefile
==============================================================================
--- head/mail/dovecot/Makefile Tue Aug 15 00:44:51 2017 (r447974)
+++ head/mail/dovecot/Makefile Tue Aug 15 01:49:01 2017 (r447975)
@@ -13,7 +13,7 @@
PORTNAME= dovecot
PORTVERSION= 2.2.31
-PORTREVISION= 1
+PORTREVISION= 2
CATEGORIES= mail ipv6
MASTER_SITES= https://www.dovecot.org/releases/2.2/
Added: head/mail/dovecot/files/patch-UPSTREAM-indexing
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ head/mail/dovecot/files/patch-UPSTREAM-indexing Tue Aug 15 01:49:01 2017 (r447975)
@@ -0,0 +1,292 @@
+From 892d85b7c6c36eb3ae6d0a78e4028a5b88b9f3a5 Mon Sep 17 00:00:00 2001
+From: Timo Sirainen <timo.sirainen at dovecot.fi>
+Date: Sat, 5 Aug 2017 14:07:58 +0900
+Subject: [PATCH] lib-index: Fix modseq tracking for
+ MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ
+
+This is used to increase modseq for mails when their private flags change.
+Use an already existing MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL() that
+does this properly.
+
+(This change preserves another bug, which is fixed in the next commit.)
+---
+ src/lib-index/mail-transaction-log-file.c | 7 +------
+ 1 file changed, 1 insertion(+), 6 deletions(-)
+
+diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
+index ab412d6ca..520541b8a 100644
+--- src/lib-index/mail-transaction-log-file.c
++++ src/lib-index/mail-transaction-log-file.c
+@@ -1013,9 +1013,6 @@ static bool
+ flag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
+ unsigned int count, unsigned int version)
+ {
+- const uint8_t internal_flags =
+- MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
+-
+ /* Hide internal flags from modseqs if the log file's version
+ is new enough. This allows upgrading without the modseqs suddenly
+ shrinking. */
+@@ -1023,9 +1020,7 @@ flag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
+ return TRUE;
+
+ for (unsigned int i = 0; i < count; i++) {
+- uint8_t changed_flags = u->add_flags | u->remove_flags;
+-
+- if ((changed_flags & ~internal_flags) != 0)
++ if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u))
+ return TRUE;
+ }
+ return FALSE;
+From ace341ac567376f37ded043c4c0f2c46b9aaecb1 Mon Sep 17 00:00:00 2001
+From: Timo Sirainen <timo.sirainen at dovecot.fi>
+Date: Sat, 5 Aug 2017 14:11:17 +0900
+Subject: [PATCH] lib-index: Fix modseq tracking with multiple flag updates
+
+The earlier code was checking only the first flag record update. If the
+first one had only internal flag changes but (some of) the rest didn't,
+the modseq wasn't counted correctly. This was probably pretty rare.
+---
+ src/lib-index/mail-transaction-log-file.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
+index 520541b8a..094cbb2d5 100644
+--- src/lib-index/mail-transaction-log-file.c
++++ src/lib-index/mail-transaction-log-file.c
+@@ -1020,7 +1020,7 @@ flag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
+ return TRUE;
+
+ for (unsigned int i = 0; i < count; i++) {
+- if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u))
++ if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(&u[i]))
+ return TRUE;
+ }
+ return FALSE;
+From 7e78f1cfc10cc0540134b5507e08524a0fdd5c93 Mon Sep 17 00:00:00 2001
+From: Timo Sirainen <timo.sirainen at dovecot.fi>
+Date: Tue, 8 Aug 2017 16:54:42 +0300
+Subject: [PATCH] lib-index: Code cleanup - move code to
+ get_modseq_next_offset_at()
+
+In preparation for making the next commit smaller.
+---
+ src/lib-index/mail-transaction-log-file.c | 80 ++++++++++++++++++-------------
+ 1 file changed, 47 insertions(+), 33 deletions(-)
+
+diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
+index 094cbb2d5..b0512e027 100644
+--- src/lib-index/mail-transaction-log-file.c
++++ src/lib-index/mail-transaction-log-file.c
+@@ -1255,15 +1255,57 @@ int mail_transaction_log_file_get_highest_modseq_at(
+ return 0;
+ }
+
++static int
++get_modseq_next_offset_at(struct mail_transaction_log_file *file,
++ uint64_t modseq,
++ uoff_t *cur_offset, uint64_t *cur_modseq,
++ uoff_t *next_offset_r)
++{
++ const struct mail_transaction_header *hdr;
++ const char *reason;
++ int ret;
++
++ /* make sure we've read until end of file. this is especially important
++ with non-head logs which might only have been opened without being
++ synced. */
++ ret = mail_transaction_log_file_map(file, *cur_offset, (uoff_t)-1, &reason);
++ if (ret <= 0) {
++ mail_index_set_error(file->log->index,
++ "Failed to map transaction log %s for getting offset "
++ "for modseq=%llu with start_offset=%"PRIuUOFF_T": %s",
++ file->filepath, (unsigned long long)modseq,
++ *cur_offset, reason);
++ return -1;
++ }
++
++ /* check sync_highest_modseq again in case sync_offset was updated */
++ if (modseq >= file->sync_highest_modseq) {
++ *next_offset_r = file->sync_offset;
++ return 0;
++ }
++
++ i_assert(*cur_offset >= file->buffer_offset);
++ while (*cur_offset < file->sync_offset) {
++ if (log_get_synced_record(file, cur_offset, &hdr, &reason) < 0) {
++ mail_index_set_error(file->log->index,
++ "%s: %s", file->filepath, reason);
++ return -1;
++ }
++ mail_transaction_update_modseq(hdr, hdr + 1, cur_modseq,
++ MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
++ if (*cur_modseq >= modseq)
++ break;
++ }
++ return 1;
++}
++
+ int mail_transaction_log_file_get_modseq_next_offset(
+ struct mail_transaction_log_file *file,
+ uint64_t modseq, uoff_t *next_offset_r)
+ {
+- const struct mail_transaction_header *hdr;
+ struct modseq_cache *cache;
+ uoff_t cur_offset;
+ uint64_t cur_modseq;
+- const char *reason;
+ int ret;
+
+ if (modseq == file->sync_highest_modseq) {
+@@ -1290,37 +1332,9 @@ int mail_transaction_log_file_get_modseq_next_offset(
+ cur_modseq = cache->highest_modseq;
+ }
+
+- /* make sure we've read until end of file. this is especially important
+- with non-head logs which might only have been opened without being
+- synced. */
+- ret = mail_transaction_log_file_map(file, cur_offset, (uoff_t)-1, &reason);
+- if (ret <= 0) {
+- mail_index_set_error(file->log->index,
+- "Failed to map transaction log %s for getting offset "
+- "for modseq=%llu with start_offset=%"PRIuUOFF_T": %s",
+- file->filepath, (unsigned long long)modseq,
+- cur_offset, reason);
+- return -1;
+- }
+-
+- /* check sync_highest_modseq again in case sync_offset was updated */
+- if (modseq >= file->sync_highest_modseq) {
+- *next_offset_r = file->sync_offset;
+- return 0;
+- }
+-
+- i_assert(cur_offset >= file->buffer_offset);
+- while (cur_offset < file->sync_offset) {
+- if (log_get_synced_record(file, &cur_offset, &hdr, &reason) < 0) {
+- mail_index_set_error(file->log->index,
+- "%s: %s", file->filepath, reason);
+- return -1;
+- }
+- mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
+- MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
+- if (cur_modseq >= modseq)
+- break;
+- }
++ if ((ret = get_modseq_next_offset_at(file, modseq, &cur_offset,
++ &cur_modseq, next_offset_r)) <= 0)
++ return ret;
+ if (cur_offset == file->sync_offset) {
+ /* if we got to sync_offset, cur_modseq should be
+ sync_highest_modseq */
+From f39bde1e1fb00905b71a4f2a793faea4b915e27b Mon Sep 17 00:00:00 2001
+From: Timo Sirainen <timo.sirainen at dovecot.fi>
+Date: Tue, 8 Aug 2017 16:56:02 +0300
+Subject: [PATCH] lib-index: Fix wrong mail_index_modseq_header automatically
+
+It happens only on the next sync, although that isn't actually guaranteed to
+happen. Still, it happens almost always so this should be good enough.
+---
+ src/lib-index/mail-index-sync.c | 2 +-
+ src/lib-index/mail-transaction-log-file.c | 26 ++++++++++++++++-----
+ src/lib-index/mail-transaction-log-private.h | 1 +
+ src/lib-index/mail-transaction-log.c | 3 +++
+ src/lib-index/test-mail-transaction-log-file.c | 32 ++++++++++++++++++++++++++
+ 5 files changed, 57 insertions(+), 7 deletions(-)
+
+diff --git a/src/lib-index/mail-index-sync.c b/src/lib-index/mail-index-sync.c
+index 24de5e119..f4d30eb03 100644
+--- src/lib-index/mail-index-sync.c
++++ src/lib-index/mail-index-sync.c
+@@ -361,7 +361,7 @@ mail_index_sync_begin_init(struct mail_index *index,
+ }
+
+ if (!mail_index_need_sync(index, flags, log_file_seq, log_file_offset) &&
+- !index->index_deleted) {
++ !index->index_deleted && !index->need_recreate) {
+ if (locked)
+ mail_transaction_log_sync_unlock(index->log, "syncing determined unnecessary");
+ return 0;
+diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
+index 8410865a1..82723fe3a 100644
+--- src/lib-index/mail-transaction-log-file.c
++++ src/lib-index/mail-transaction-log-file.c
+@@ -1257,7 +1257,7 @@ int mail_transaction_log_file_get_highest_modseq_at(
+
+ static int
+ get_modseq_next_offset_at(struct mail_transaction_log_file *file,
+- uint64_t modseq,
++ uint64_t modseq, bool use_highest,
+ uoff_t *cur_offset, uint64_t *cur_modseq,
+ uoff_t *next_offset_r)
+ {
+@@ -1279,7 +1279,7 @@ get_modseq_next_offset_at(struct mail_transaction_log_file *file,
+ }
+
+ /* check sync_highest_modseq again in case sync_offset was updated */
+- if (modseq >= file->sync_highest_modseq) {
++ if (modseq >= file->sync_highest_modseq && use_highest) {
+ *next_offset_r = file->sync_offset;
+ return 0;
+ }
+@@ -1332,16 +1332,30 @@ int mail_transaction_log_file_get_modseq_next_offset(
+ cur_modseq = cache->highest_modseq;
+ }
+
+- if ((ret = get_modseq_next_offset_at(file, modseq, &cur_offset,
++ if ((ret = get_modseq_next_offset_at(file, modseq, TRUE, &cur_offset,
+ &cur_modseq, next_offset_r)) <= 0)
+ return ret;
+ if (cur_offset == file->sync_offset) {
+ /* if we got to sync_offset, cur_modseq should be
+ sync_highest_modseq */
+ mail_index_set_error(file->log->index,
+- "%s: Transaction log changed unexpectedly, "
+- "can't get modseq", file->filepath);
+- return -1;
++ "%s: Transaction log modseq tracking is corrupted - fixing",
++ file->filepath);
++ /* retry getting the offset by reading from the beginning
++ of the file */
++ cur_offset = file->hdr.hdr_size;
++ cur_modseq = file->hdr.initial_modseq;
++ ret = get_modseq_next_offset_at(file, modseq, FALSE,
++ &cur_offset, &cur_modseq,
++ next_offset_r);
++ if (ret < 0)
++ return -1;
++ i_assert(ret != 0);
++ /* get it fixed on the next sync */
++ file->log->index->need_recreate = TRUE;
++ file->need_rotate = TRUE;
++ /* clear cache, since it's unreliable */
++ memset(file->modseq_cache, 0, sizeof(file->modseq_cache));
+ }
+
+ /* @UNSAFE: cache the value */
+diff --git a/src/lib-index/mail-transaction-log-private.h b/src/lib-index/mail-transaction-log-private.h
+index f56434b40..cba0b6b5b 100644
+--- src/lib-index/mail-transaction-log-private.h
++++ src/lib-index/mail-transaction-log-private.h
+@@ -81,6 +81,7 @@ struct mail_transaction_log_file {
+ unsigned int locked:1;
+ unsigned int locked_sync_offset_updated:1;
+ unsigned int corrupted:1;
++ unsigned int need_rotate:1;
+ };
+
+ struct mail_transaction_log {
+diff --git a/src/lib-index/mail-transaction-log.c b/src/lib-index/mail-transaction-log.c
+index 1738e367f..16d301f34 100644
+--- src/lib-index/mail-transaction-log.c
++++ src/lib-index/mail-transaction-log.c
+@@ -234,6 +234,9 @@ bool mail_transaction_log_want_rotate(struct mail_transaction_log *log)
+ {
+ struct mail_transaction_log_file *file = log->head;
+
++ if (file->need_rotate)
++ return TRUE;
++
+ if (file->hdr.major_version < MAIL_TRANSACTION_LOG_MAJOR_VERSION ||
+ (file->hdr.major_version == MAIL_TRANSACTION_LOG_MAJOR_VERSION &&
+ file->hdr.minor_version < MAIL_TRANSACTION_LOG_MINOR_VERSION)) {
More information about the svn-ports-all
mailing list