MySQL 5.6.14 Source Code Document
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
fts0opt.cc
Go to the documentation of this file.
1 /*****************************************************************************
2 
3 Copyright (c) 2007, 2013, Oracle and/or its affiliates. All Rights Reserved.
4 
5 This program is free software; you can redistribute it and/or modify it under
6 the terms of the GNU General Public License as published by the Free Software
7 Foundation; version 2 of the License.
8 
9 This program is distributed in the hope that it will be useful, but WITHOUT
10 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
11 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
12 
13 You should have received a copy of the GNU General Public License along with
14 this program; if not, write to the Free Software Foundation, Inc.,
15 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA
16 
17 *****************************************************************************/
18 
19 /******************************************************************/
28 #include "fts0fts.h"
29 #include "row0sel.h"
30 #include "que0types.h"
31 #include "fts0priv.h"
32 #include "fts0types.h"
33 #include "ut0wqueue.h"
34 #include "srv0start.h"
35 #include "zlib.h"
36 
37 #ifndef UNIV_NONINL
38 #include "fts0types.ic"
39 #include "fts0vlc.ic"
40 #endif
41 
43 static ib_wqueue_t* fts_optimize_wq;
44 
46 static const ulint FTS_MAX_DELETE_DOC_IDS = 1000;
47 
49 static const ulint FTS_QUEUE_WAIT_IN_USECS = 5000000;
50 
52 static const ulint FTS_OPTIMIZE_INTERVAL_IN_SECS = 300;
53 
55 static bool fts_opt_start_shutdown = false;
56 
58 static ib_time_t last_check_sync_time;
59 
60 #if 0
61 
63 static ulint fts_optimize_sync_iterator = 0;
64 #endif
65 
67 enum fts_state_t {
68  FTS_STATE_LOADED,
69  FTS_STATE_RUNNING,
70  FTS_STATE_SUSPENDED,
71  FTS_STATE_DONE,
72  FTS_STATE_EMPTY
73 };
74 
76 enum fts_msg_type_t {
77  FTS_MSG_START,
79  FTS_MSG_PAUSE,
81  FTS_MSG_STOP,
83  FTS_MSG_ADD_TABLE,
86  FTS_MSG_OPTIMIZE_TABLE,
88  FTS_MSG_DEL_TABLE,
90 };
91 
94 struct fts_zip_t {
95  ulint status;
97  ulint n_words;
99  ulint block_sz;
101  ib_vector_t* blocks;
103  ib_alloc_t* heap_alloc;
105  ulint pos;
107  ulint last_big_block;
112  z_streamp zp;
118  fts_string_t word;
120  ulint max_words;
122 };
123 
125 struct fts_optimize_graph_t {
127  que_t* delete_nodes_graph;
129  que_t* write_nodes_graph;
131  que_t* commit_graph;
133  que_t* read_nodes_graph;
134 };
135 
137 struct fts_optimize_t {
138  trx_t* trx;
140  ib_alloc_t* self_heap;
142  char* name_prefix;
144  fts_table_t fts_index_table;
147  fts_table_t fts_common_table;
148 
149  dict_table_t* table;
151  dict_index_t* index;
153  fts_doc_ids_t* to_delete;
159  ulint del_pos;
163  ibool done;
165  ib_vector_t* words;
168  fts_zip_t* zip;
170  fts_optimize_graph_t
171  graph; /*optimize */
172 
173  ulint n_completed;
175  ibool del_list_regenerated;
177 };
178 
180 struct fts_encode_t {
181  doc_id_t src_last_doc_id;
182  byte* src_ilist_ptr;
183 };
184 
187 struct fts_slot_t {
188  dict_table_t* table;
190  fts_state_t state;
192  ulint added;
195  ulint deleted;
198  ib_time_t last_run;
200  ib_time_t completed;
202  ib_time_t interval_time;
204 };
205 
207 struct fts_msg_del_t {
208  dict_table_t* table;
210  os_event_t event;
213 };
214 
216 struct fts_msg_optimize_t {
217  dict_table_t* table;
218 };
219 
221 struct fts_msg_t {
222  fts_msg_type_t type;
224  void* ptr;
226  mem_heap_t* heap;
229 };
230 
232 UNIV_INTERN ulong fts_num_word_optimize;
233 
234 // FIXME
235 UNIV_INTERN char fts_enable_diag_print;
236 
238 static ulint FTS_ZIP_BLOCK_SIZE = 1024;
239 
241 static ib_time_t fts_optimize_time_limit = 0;
242 
244 static const char* fts_init_delete_sql =
245  "BEGIN\n"
246  "\n"
247  "INSERT INTO %s_BEING_DELETED\n"
248  "SELECT doc_id FROM \"%s_DELETED\";\n"
249  "\n"
250  "INSERT INTO %s_BEING_DELETED_CACHE\n"
251  "SELECT doc_id FROM \"%s_DELETED_CACHE\";\n";
252 
253 static const char* fts_delete_doc_ids_sql =
254  "BEGIN\n"
255  "\n"
256  "DELETE FROM \"%s_DELETED\" WHERE doc_id = :doc_id1;\n"
257  "DELETE FROM \"%s_DELETED_CACHE\" WHERE doc_id = :doc_id2;\n";
258 
259 static const char* fts_end_delete_sql =
260  "BEGIN\n"
261  "\n"
262  "DELETE FROM \"%s_BEING_DELETED\";\n"
263  "DELETE FROM \"%s_BEING_DELETED_CACHE\";\n";
264 
265 /**********************************************************************/
267 static
268 void
269 fts_zip_initialize(
270 /*===============*/
271  fts_zip_t* zip)
272 {
273  zip->pos = 0;
274  zip->n_words = 0;
275 
276  zip->status = Z_OK;
277 
278  zip->last_big_block = 0;
279 
280  zip->word.f_len = 0;
281  memset(zip->word.f_str, 0, FTS_MAX_WORD_LEN);
282 
283  ib_vector_reset(zip->blocks);
284 
285  memset(zip->zp, 0, sizeof(*zip->zp));
286 }
287 
288 /**********************************************************************/
291 static
292 fts_zip_t*
293 fts_zip_create(
294 /*===========*/
295  mem_heap_t* heap,
296  ulint block_sz,
297  ulint max_words)
298 {
299  fts_zip_t* zip;
300 
301  zip = static_cast<fts_zip_t*>(mem_heap_zalloc(heap, sizeof(*zip)));
302 
303  zip->word.f_str = static_cast<byte*>(
304  mem_heap_zalloc(heap, FTS_MAX_WORD_LEN + 1));
305 
306  zip->block_sz = block_sz;
307 
308  zip->heap_alloc = ib_heap_allocator_create(heap);
309 
310  zip->blocks = ib_vector_create(zip->heap_alloc, sizeof(void*), 128);
311 
312  zip->max_words = max_words;
313 
314  zip->zp = static_cast<z_stream*>(
315  mem_heap_zalloc(heap, sizeof(*zip->zp)));
316 
317  return(zip);
318 }
319 
320 /**********************************************************************/
322 static
323 void
324 fts_zip_init(
325 /*=========*/
326 
327  fts_zip_t* zip)
328 {
329  memset(zip->zp, 0, sizeof(*zip->zp));
330 
331  zip->word.f_len = 0;
332  *zip->word.f_str = '\0';
333 }
334 
335 /**********************************************************************/
338 UNIV_INTERN
339 fts_word_t*
340 fts_word_init(
341 /*==========*/
342  fts_word_t* word,
343  byte* utf8,
344  ulint len)
345 {
346  mem_heap_t* heap = mem_heap_create(sizeof(fts_node_t));
347 
348  memset(word, 0, sizeof(*word));
349 
350  word->text.f_len = len;
351  word->text.f_str = static_cast<byte*>(mem_heap_alloc(heap, len + 1));
352 
353  /* Need to copy the NUL character too. */
354  memcpy(word->text.f_str, utf8, word->text.f_len);
355  word->text.f_str[word->text.f_len] = 0;
356 
357  word->heap_alloc = ib_heap_allocator_create(heap);
358 
359  word->nodes = ib_vector_create(
360  word->heap_alloc, sizeof(fts_node_t), 64);
361 
362  return(word);
363 }
364 
365 /**********************************************************************/
368 static
369 fts_node_t*
370 fts_optimize_read_node(
371 /*===================*/
372  fts_word_t* word,
373  que_node_t* exp)
374 {
375  int i;
376  fts_node_t* node = static_cast<fts_node_t*>(
377  ib_vector_push(word->nodes, NULL));
378 
379  /* Start from 1 since the first node has been read by the caller */
380  for (i = 1; exp; exp = que_node_get_next(exp), ++i) {
381 
382  dfield_t* dfield = que_node_get_val(exp);
383  byte* data = static_cast<byte*>(
384  dfield_get_data(dfield));
385  ulint len = dfield_get_len(dfield);
386 
387  ut_a(len != UNIV_SQL_NULL);
388 
389  /* Note: The column numbers below must match the SELECT */
390  switch (i) {
391  case 1: /* DOC_COUNT */
392  node->doc_count = mach_read_from_4(data);
393  break;
394 
395  case 2: /* FIRST_DOC_ID */
396  node->first_doc_id = fts_read_doc_id(data);
397  break;
398 
399  case 3: /* LAST_DOC_ID */
400  node->last_doc_id = fts_read_doc_id(data);
401  break;
402 
403  case 4: /* ILIST */
404  node->ilist_size_alloc = node->ilist_size = len;
405  node->ilist = static_cast<byte*>(ut_malloc(len));
406  memcpy(node->ilist, data, len);
407  break;
408 
409  default:
410  ut_error;
411  }
412  }
413 
414  /* Make sure all columns were read. */
415  ut_a(i == 5);
416 
417  return(node);
418 }
419 
420 /**********************************************************************/
423 UNIV_INTERN
424 ibool
425 fts_optimize_index_fetch_node(
426 /*==========================*/
427  void* row,
428  void* user_arg)
429 {
430  fts_word_t* word;
431  sel_node_t* sel_node = static_cast<sel_node_t*>(row);
432  fts_fetch_t* fetch = static_cast<fts_fetch_t*>(user_arg);
433  ib_vector_t* words = static_cast<ib_vector_t*>(fetch->read_arg);
434  que_node_t* exp = sel_node->select_list;
435  dfield_t* dfield = que_node_get_val(exp);
436  void* data = dfield_get_data(dfield);
437  ulint dfield_len = dfield_get_len(dfield);
438 
439  ut_a(dfield_len <= FTS_MAX_WORD_LEN);
440 
441  if (ib_vector_size(words) == 0) {
442 
443  word = static_cast<fts_word_t*>(ib_vector_push(words, NULL));
444  fts_word_init(word, (byte*) data, dfield_len);
445  }
446 
447  word = static_cast<fts_word_t*>(ib_vector_last(words));
448 
449  if (dfield_len != word->text.f_len
450  || memcmp(word->text.f_str, data, dfield_len)) {
451 
452  word = static_cast<fts_word_t*>(ib_vector_push(words, NULL));
453  fts_word_init(word, (byte*) data, dfield_len);
454  }
455 
456  fts_optimize_read_node(word, que_node_get_next(exp));
457 
458  return(TRUE);
459 }
460 
461 /**********************************************************************/
464 UNIV_INTERN
465 dberr_t
466 fts_index_fetch_nodes(
467 /*==================*/
468  trx_t* trx,
469  que_t** graph,
470  fts_table_t* fts_table,
471  const fts_string_t*
472  word,
473  fts_fetch_t* fetch)
474 {
475  pars_info_t* info;
476  dberr_t error;
477 
478  trx->op_info = "fetching FTS index nodes";
479 
480  if (*graph) {
481  info = (*graph)->info;
482  } else {
483  info = pars_info_create();
484  }
485 
486  pars_info_bind_function(info, "my_func", fetch->read_record, fetch);
487  pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len);
488 
489  if (!*graph) {
490  ulint selected;
491 
492  ut_a(fts_table->type == FTS_INDEX_TABLE);
493 
494  selected = fts_select_index(fts_table->charset,
495  word->f_str, word->f_len);
496 
497  fts_table->suffix = fts_get_suffix(selected);
498 
499  *graph = fts_parse_sql(
500  fts_table,
501  info,
502  "DECLARE FUNCTION my_func;\n"
503  "DECLARE CURSOR c IS"
504  " SELECT word, doc_count, first_doc_id, last_doc_id, "
505  "ilist\n"
506  " FROM \"%s\"\n"
507  " WHERE word LIKE :word\n"
508  " ORDER BY first_doc_id;\n"
509  "BEGIN\n"
510  "\n"
511  "OPEN c;\n"
512  "WHILE 1 = 1 LOOP\n"
513  " FETCH c INTO my_func();\n"
514  " IF c % NOTFOUND THEN\n"
515  " EXIT;\n"
516  " END IF;\n"
517  "END LOOP;\n"
518  "CLOSE c;");
519  }
520 
521  for(;;) {
522  error = fts_eval_sql(trx, *graph);
523 
524  if (error == DB_SUCCESS) {
525  fts_sql_commit(trx);
526 
527  break; /* Exit the loop. */
528  } else {
529  fts_sql_rollback(trx);
530 
531  ut_print_timestamp(stderr);
532 
533  if (error == DB_LOCK_WAIT_TIMEOUT) {
534  fprintf(stderr, " InnoDB: Warning: lock wait "
535  "timeout reading FTS index. "
536  "Retrying!\n");
537 
538  trx->error_state = DB_SUCCESS;
539  } else {
540  fprintf(stderr, " InnoDB: Error: (%s) "
541  "while reading FTS index.\n",
542  ut_strerr(error));
543 
544  break; /* Exit the loop. */
545  }
546  }
547  }
548 
549  return(error);
550 }
551 
552 /**********************************************************************/
554 static
555 byte*
556 fts_zip_read_word(
557 /*==============*/
558  fts_zip_t* zip,
559  fts_string_t* word)
560 {
561 #ifdef UNIV_DEBUG
562  ulint i;
563 #endif
564  byte len = 0;
565  void* null = NULL;
566  byte* ptr = word->f_str;
567  int flush = Z_NO_FLUSH;
568 
569  /* Either there was an error or we are at the Z_STREAM_END. */
570  if (zip->status != Z_OK) {
571  return(NULL);
572  }
573 
574  zip->zp->next_out = &len;
575  zip->zp->avail_out = sizeof(len);
576 
577  while (zip->status == Z_OK && zip->zp->avail_out > 0) {
578 
579  /* Finished decompressing block. */
580  if (zip->zp->avail_in == 0) {
581 
582  /* Free the block thats been decompressed. */
583  if (zip->pos > 0) {
584  ulint prev = zip->pos - 1;
585 
586  ut_a(zip->pos < ib_vector_size(zip->blocks));
587 
588  ut_free(ib_vector_getp(zip->blocks, prev));
589  ib_vector_set(zip->blocks, prev, &null);
590  }
591 
592  /* Any more blocks to decompress. */
593  if (zip->pos < ib_vector_size(zip->blocks)) {
594 
595  zip->zp->next_in = static_cast<byte*>(
596  ib_vector_getp(
597  zip->blocks, zip->pos));
598 
599  if (zip->pos > zip->last_big_block) {
600  zip->zp->avail_in =
601  FTS_MAX_WORD_LEN;
602  } else {
603  zip->zp->avail_in = zip->block_sz;
604  }
605 
606  ++zip->pos;
607  } else {
608  flush = Z_FINISH;
609  }
610  }
611 
612  switch (zip->status = inflate(zip->zp, flush)) {
613  case Z_OK:
614  if (zip->zp->avail_out == 0 && len > 0) {
615 
616  ut_a(len <= FTS_MAX_WORD_LEN);
617  ptr[len] = 0;
618 
619  zip->zp->next_out = ptr;
620  zip->zp->avail_out = len;
621 
622  word->f_len = len;
623  len = 0;
624  }
625  break;
626 
627  case Z_BUF_ERROR: /* No progress possible. */
628  case Z_STREAM_END:
629  inflateEnd(zip->zp);
630  break;
631 
632  case Z_STREAM_ERROR:
633  default:
634  ut_error;
635  }
636  }
637 
638 #ifdef UNIV_DEBUG
639  /* All blocks must be freed at end of inflate. */
640  if (zip->status != Z_OK) {
641  for (i = 0; i < ib_vector_size(zip->blocks); ++i) {
642  if (ib_vector_getp(zip->blocks, i)) {
643  ut_free(ib_vector_getp(zip->blocks, i));
644  ib_vector_set(zip->blocks, i, &null);
645  }
646  }
647  }
648 
649  if (ptr != NULL) {
650  ut_ad(word->f_len == strlen((char*) ptr));
651  }
652 #endif /* UNIV_DEBUG */
653 
654  return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL);
655 }
656 
657 /**********************************************************************/
661 static
662 ibool
663 fts_fetch_index_words(
664 /*==================*/
665  void* row,
666  void* user_arg)
667 {
668  sel_node_t* sel_node = static_cast<sel_node_t*>(row);
669  fts_zip_t* zip = static_cast<fts_zip_t*>(user_arg);
670  que_node_t* exp = sel_node->select_list;
671  dfield_t* dfield = que_node_get_val(exp);
672  byte len = (byte) dfield_get_len(dfield);
673  void* data = dfield_get_data(dfield);
674 
675  /* Skip the duplicate words. */
676  if (zip->word.f_len == len && !memcmp(zip->word.f_str, data, len)) {
677 
678  return(TRUE);
679  }
680 
681  ut_a(len <= FTS_MAX_WORD_LEN);
682 
683  memcpy(zip->word.f_str, data, len);
684  zip->word.f_len = len;
685 
686  ut_a(zip->zp->avail_in == 0);
687  ut_a(zip->zp->next_in == NULL);
688 
689  /* The string is prefixed by len. */
690  zip->zp->next_in = &len;
691  zip->zp->avail_in = sizeof(len);
692 
693  /* Compress the word, create output blocks as necessary. */
694  while (zip->zp->avail_in > 0) {
695 
696  /* No space left in output buffer, create a new one. */
697  if (zip->zp->avail_out == 0) {
698  byte* block;
699 
700  block = static_cast<byte*>(ut_malloc(zip->block_sz));
701  ib_vector_push(zip->blocks, &block);
702 
703  zip->zp->next_out = block;
704  zip->zp->avail_out = zip->block_sz;
705  }
706 
707  switch (zip->status = deflate(zip->zp, Z_NO_FLUSH)) {
708  case Z_OK:
709  if (zip->zp->avail_in == 0) {
710  zip->zp->next_in = static_cast<byte*>(data);
711  zip->zp->avail_in = len;
712  ut_a(len <= FTS_MAX_WORD_LEN);
713  len = 0;
714  }
715  break;
716 
717  case Z_STREAM_END:
718  case Z_BUF_ERROR:
719  case Z_STREAM_ERROR:
720  default:
721  ut_error;
722  break;
723  }
724  }
725 
726  /* All data should have been compressed. */
727  ut_a(zip->zp->avail_in == 0);
728  zip->zp->next_in = NULL;
729 
730  ++zip->n_words;
731 
732  return(zip->n_words >= zip->max_words ? FALSE : TRUE);
733 }
734 
735 /**********************************************************************/
737 static
738 void
739 fts_zip_deflate_end(
740 /*================*/
741  fts_zip_t* zip)
742 {
743  ut_a(zip->zp->avail_in == 0);
744  ut_a(zip->zp->next_in == NULL);
745 
746  zip->status = deflate(zip->zp, Z_FINISH);
747 
748  ut_a(ib_vector_size(zip->blocks) > 0);
749  zip->last_big_block = ib_vector_size(zip->blocks) - 1;
750 
751  /* Allocate smaller block(s), since this is trailing data. */
752  while (zip->status == Z_OK) {
753  byte* block;
754 
755  ut_a(zip->zp->avail_out == 0);
756 
757  block = static_cast<byte*>(ut_malloc(FTS_MAX_WORD_LEN + 1));
758  ib_vector_push(zip->blocks, &block);
759 
760  zip->zp->next_out = block;
761  zip->zp->avail_out = FTS_MAX_WORD_LEN;
762 
763  zip->status = deflate(zip->zp, Z_FINISH);
764  }
765 
766  ut_a(zip->status == Z_STREAM_END);
767 
768  zip->status = deflateEnd(zip->zp);
769  ut_a(zip->status == Z_OK);
770 
771  /* Reset the ZLib data structure. */
772  memset(zip->zp, 0, sizeof(*zip->zp));
773 }
774 
775 /**********************************************************************/
779 static __attribute__((nonnull, warn_unused_result))
780 dberr_t
781 fts_index_fetch_words(
782 /*==================*/
783  fts_optimize_t* optim,
784  const fts_string_t* word,
786  ulint n_words)
787 {
788  pars_info_t* info;
789  que_t* graph;
790  ulint selected;
791  fts_zip_t* zip = NULL;
792  dberr_t error = DB_SUCCESS;
793  mem_heap_t* heap = static_cast<mem_heap_t*>(optim->self_heap->arg);
794  ibool inited = FALSE;
795 
796  optim->trx->op_info = "fetching FTS index words";
797 
798  if (optim->zip == NULL) {
799  optim->zip = fts_zip_create(heap, FTS_ZIP_BLOCK_SIZE, n_words);
800  } else {
801  fts_zip_initialize(optim->zip);
802  }
803 
804  for (selected = fts_select_index(
805  optim->fts_index_table.charset, word->f_str, word->f_len);
806  fts_index_selector[selected].value;
807  selected++) {
808 
809  optim->fts_index_table.suffix = fts_get_suffix(selected);
810 
811  /* We've search all indexes. */
812  if (optim->fts_index_table.suffix == NULL) {
813  return(DB_TABLE_NOT_FOUND);
814  }
815 
816  info = pars_info_create();
817 
818  pars_info_bind_function(
819  info, "my_func", fts_fetch_index_words, optim->zip);
820 
821  pars_info_bind_varchar_literal(
822  info, "word", word->f_str, word->f_len);
823 
824  graph = fts_parse_sql(
825  &optim->fts_index_table,
826  info,
827  "DECLARE FUNCTION my_func;\n"
828  "DECLARE CURSOR c IS"
829  " SELECT word\n"
830  " FROM \"%s\"\n"
831  " WHERE word > :word\n"
832  " ORDER BY word;\n"
833  "BEGIN\n"
834  "\n"
835  "OPEN c;\n"
836  "WHILE 1 = 1 LOOP\n"
837  " FETCH c INTO my_func();\n"
838  " IF c % NOTFOUND THEN\n"
839  " EXIT;\n"
840  " END IF;\n"
841  "END LOOP;\n"
842  "CLOSE c;");
843 
844  zip = optim->zip;
845 
846  for(;;) {
847  int err;
848 
849  if (!inited && ((err = deflateInit(zip->zp, 9))
850  != Z_OK)) {
851  ut_print_timestamp(stderr);
852  fprintf(stderr,
853  " InnoDB: Error: ZLib deflateInit() "
854  "failed: %d\n", err);
855 
856  error = DB_ERROR;
857  break;
858  } else {
859  inited = TRUE;
860  error = fts_eval_sql(optim->trx, graph);
861  }
862 
863  if (error == DB_SUCCESS) {
864  //FIXME fts_sql_commit(optim->trx);
865  break;
866  } else {
867  //FIXME fts_sql_rollback(optim->trx);
868 
869  ut_print_timestamp(stderr);
870 
871  if (error == DB_LOCK_WAIT_TIMEOUT) {
872  fprintf(stderr, " InnoDB: "
873  "Warning: lock wait "
874  "timeout reading document. "
875  "Retrying!\n");
876 
877  /* We need to reset the ZLib state. */
878  inited = FALSE;
879  deflateEnd(zip->zp);
880  fts_zip_init(zip);
881 
882  optim->trx->error_state = DB_SUCCESS;
883  } else {
884  fprintf(stderr, " InnoDB: Error: (%s) "
885  "while reading document.\n",
886  ut_strerr(error));
887 
888  break; /* Exit the loop. */
889  }
890  }
891  }
892 
893  fts_que_graph_free(graph);
894 
895  /* Check if max word to fetch is exceeded */
896  if (optim->zip->n_words >= n_words) {
897  break;
898  }
899  }
900 
901  if (error == DB_SUCCESS && zip->status == Z_OK && zip->n_words > 0) {
902 
903  /* All data should have been read. */
904  ut_a(zip->zp->avail_in == 0);
905 
906  fts_zip_deflate_end(zip);
907  } else {
908  deflateEnd(zip->zp);
909  }
910 
911  return(error);
912 }
913 
914 /**********************************************************************/
917 static
918 ibool
919 fts_fetch_doc_ids(
920 /*==============*/
921  void* row,
922  void* user_arg)
923 {
924  que_node_t* exp;
925  int i = 0;
926  sel_node_t* sel_node = static_cast<sel_node_t*>(row);
927  fts_doc_ids_t* fts_doc_ids = static_cast<fts_doc_ids_t*>(user_arg);
928  fts_update_t* update = static_cast<fts_update_t*>(
929  ib_vector_push(fts_doc_ids->doc_ids, NULL));
930 
931  for (exp = sel_node->select_list;
932  exp;
933  exp = que_node_get_next(exp), ++i) {
934 
935  dfield_t* dfield = que_node_get_val(exp);
936  void* data = dfield_get_data(dfield);
937  ulint len = dfield_get_len(dfield);
938 
939  ut_a(len != UNIV_SQL_NULL);
940 
941  /* Note: The column numbers below must match the SELECT. */
942  switch (i) {
943  case 0: /* DOC_ID */
944  update->fts_indexes = NULL;
945  update->doc_id = fts_read_doc_id(
946  static_cast<byte*>(data));
947  break;
948 
949  default:
950  ut_error;
951  }
952  }
953 
954  return(TRUE);
955 }
956 
957 /**********************************************************************/
960 UNIV_INTERN
961 dberr_t
962 fts_table_fetch_doc_ids(
963 /*====================*/
964  trx_t* trx,
965  fts_table_t* fts_table,
966  fts_doc_ids_t* doc_ids)
967 {
968  dberr_t error;
969  que_t* graph;
970  pars_info_t* info = pars_info_create();
971  ibool alloc_bk_trx = FALSE;
972 
973  ut_a(fts_table->suffix != NULL);
974  ut_a(fts_table->type == FTS_COMMON_TABLE);
975 
976  if (!trx) {
977  trx = trx_allocate_for_background();
978  alloc_bk_trx = TRUE;
979  }
980 
981  trx->op_info = "fetching FTS doc ids";
982 
983  pars_info_bind_function(info, "my_func", fts_fetch_doc_ids, doc_ids);
984 
985  graph = fts_parse_sql(
986  fts_table,
987  info,
988  "DECLARE FUNCTION my_func;\n"
989  "DECLARE CURSOR c IS"
990  " SELECT doc_id FROM \"%s\";\n"
991  "BEGIN\n"
992  "\n"
993  "OPEN c;\n"
994  "WHILE 1 = 1 LOOP\n"
995  " FETCH c INTO my_func();\n"
996  " IF c % NOTFOUND THEN\n"
997  " EXIT;\n"
998  " END IF;\n"
999  "END LOOP;\n"
1000  "CLOSE c;");
1001 
1002  error = fts_eval_sql(trx, graph);
1003 
1004  mutex_enter(&dict_sys->mutex);
1005  que_graph_free(graph);
1006  mutex_exit(&dict_sys->mutex);
1007 
1008  if (error == DB_SUCCESS) {
1009  fts_sql_commit(trx);
1010 
1011  ib_vector_sort(doc_ids->doc_ids, fts_update_doc_id_cmp);
1012  } else {
1013  fts_sql_rollback(trx);
1014  }
1015 
1016  if (alloc_bk_trx) {
1017  trx_free_for_background(trx);
1018  }
1019 
1020  return(error);
1021 }
1022 
1023 /**********************************************************************/
1027 UNIV_INTERN
1028 int
1029 fts_bsearch(
1030 /*========*/
1031  fts_update_t* array,
1032  int lower,
1033  int upper,
1034  doc_id_t doc_id)
1035 {
1036  int orig_size = upper;
1037 
1038  if (upper == 0) {
1039  /* Nothing to search */
1040  return(-1);
1041  } else {
1042  while (lower < upper) {
1043  int i = (lower + upper) >> 1;
1044 
1045  if (doc_id > array[i].doc_id) {
1046  lower = i + 1;
1047  } else if (doc_id < array[i].doc_id) {
1048  upper = i - 1;
1049  } else {
1050  return(i); /* Found. */
1051  }
1052  }
1053  }
1054 
1055  if (lower == upper && lower < orig_size) {
1056  if (doc_id == array[lower].doc_id) {
1057  return(lower);
1058  } else if (lower == 0) {
1059  return(-1);
1060  }
1061  }
1062 
1063  /* Not found. */
1064  return( (lower == 0) ? -1 : -lower);
1065 }
1066 
1067 /**********************************************************************/
1072 static
1073 int
1074 fts_optimize_lookup(
1075 /*================*/
1076  ib_vector_t* doc_ids,
1077  ulint lower,
1078  doc_id_t first_doc_id,
1079  doc_id_t last_doc_id)
1080 {
1081  int pos;
1082  int upper = ib_vector_size(doc_ids);
1083  fts_update_t* array = (fts_update_t*) doc_ids->data;
1084 
1085  pos = fts_bsearch(array, lower, upper, first_doc_id);
1086 
1087  ut_a(abs(pos) <= upper + 1);
1088 
1089  if (pos < 0) {
1090 
1091  int i = abs(pos);
1092 
1093  /* If i is 1, it could be first_doc_id is less than
1094  either the first or second array item, do a
1095  double check */
1096  if (i == 1 && array[0].doc_id <= last_doc_id
1097  && first_doc_id < array[0].doc_id) {
1098  pos = 0;
1099  } else if (i < upper && array[i].doc_id <= last_doc_id) {
1100 
1101  /* Check if the "next" doc id is within the
1102  first & last doc id of the node. */
1103  pos = i;
1104  }
1105  }
1106 
1107  return(pos);
1108 }
1109 
1110 /**********************************************************************/
1113 static __attribute__((nonnull))
1114 dberr_t
1115 fts_optimize_encode_node(
1116 /*=====================*/
1117  fts_node_t* node,
1118  doc_id_t doc_id,
1119  fts_encode_t* enc)
1120 {
1121  byte* dst;
1122  ulint enc_len;
1123  ulint pos_enc_len;
1124  doc_id_t doc_id_delta;
1125  dberr_t error = DB_SUCCESS;
1126  byte* src = enc->src_ilist_ptr;
1127 
1128  if (node->first_doc_id == 0) {
1129  ut_a(node->last_doc_id == 0);
1130 
1131  node->first_doc_id = doc_id;
1132  }
1133 
1134  /* Calculate the space required to store the ilist. */
1135  doc_id_delta = doc_id - node->last_doc_id;
1136  enc_len = fts_get_encoded_len(static_cast<ulint>(doc_id_delta));
1137 
1138  /* Calculate the size of the encoded pos array. */
1139  while (*src) {
1140  fts_decode_vlc(&src);
1141  }
1142 
1143  /* Skip the 0x00 byte at the end of the word positions list. */
1144  ++src;
1145 
1146  /* Number of encoded pos bytes to copy. */
1147  pos_enc_len = src - enc->src_ilist_ptr;
1148 
1149  /* Total number of bytes required for copy. */
1150  enc_len += pos_enc_len;
1151 
1152  /* Check we have enough space in the destination buffer for
1153  copying the document word list. */
1154  if (!node->ilist) {
1155  ulint new_size;
1156 
1157  ut_a(node->ilist_size == 0);
1158 
1159  new_size = enc_len > FTS_ILIST_MAX_SIZE
1160  ? enc_len : FTS_ILIST_MAX_SIZE;
1161 
1162  node->ilist = static_cast<byte*>(ut_malloc(new_size));
1163  node->ilist_size_alloc = new_size;
1164 
1165  } else if ((node->ilist_size + enc_len) > node->ilist_size_alloc) {
1166  ulint new_size = node->ilist_size + enc_len;
1167  byte* ilist = static_cast<byte*>(ut_malloc(new_size));
1168 
1169  memcpy(ilist, node->ilist, node->ilist_size);
1170 
1171  ut_free(node->ilist);
1172 
1173  node->ilist = ilist;
1174  node->ilist_size_alloc = new_size;
1175  }
1176 
1177  src = enc->src_ilist_ptr;
1178  dst = node->ilist + node->ilist_size;
1179 
1180  /* Encode the doc id. Cast to ulint, the delta should be small and
1181  therefore no loss of precision. */
1182  dst += fts_encode_int((ulint) doc_id_delta, dst);
1183 
1184  /* Copy the encoded pos array. */
1185  memcpy(dst, src, pos_enc_len);
1186 
1187  node->last_doc_id = doc_id;
1188 
1189  /* Data copied upto here. */
1190  node->ilist_size += enc_len;
1191  enc->src_ilist_ptr += pos_enc_len;
1192 
1193  ut_a(node->ilist_size <= node->ilist_size_alloc);
1194 
1195  return(error);
1196 }
1197 
1198 /**********************************************************************/
1201 static __attribute__((nonnull))
1202 dberr_t
1203 fts_optimize_node(
1204 /*==============*/
1205  ib_vector_t* del_vec,
1206  int* del_pos,
1207  fts_node_t* dst_node,
1208  fts_node_t* src_node,
1209  fts_encode_t* enc)
1210 {
1211  ulint copied;
1212  dberr_t error = DB_SUCCESS;
1213  doc_id_t doc_id = enc->src_last_doc_id;
1214 
1215  if (!enc->src_ilist_ptr) {
1216  enc->src_ilist_ptr = src_node->ilist;
1217  }
1218 
1219  copied = enc->src_ilist_ptr - src_node->ilist;
1220 
1221  /* While there is data in the source node and space to copy
1222  into in the destination node. */
1223  while (copied < src_node->ilist_size
1224  && dst_node->ilist_size < FTS_ILIST_MAX_SIZE) {
1225 
1226  doc_id_t delta;
1227  doc_id_t del_doc_id = FTS_NULL_DOC_ID;
1228 
1229  delta = fts_decode_vlc(&enc->src_ilist_ptr);
1230 
1231 test_again:
1232  /* Check whether the doc id is in the delete list, if
1233  so then we skip the entries but we need to track the
1234  delta for decoding the entries following this document's
1235  entries. */
1236  if (*del_pos >= 0 && *del_pos < (int) ib_vector_size(del_vec)) {
1237  fts_update_t* update;
1238 
1239  update = (fts_update_t*) ib_vector_get(
1240  del_vec, *del_pos);
1241 
1242  del_doc_id = update->doc_id;
1243  }
1244 
1245  if (enc->src_ilist_ptr == src_node->ilist && doc_id == 0) {
1246  ut_a(delta == src_node->first_doc_id);
1247  }
1248 
1249  doc_id += delta;
1250 
1251  if (del_doc_id > 0 && doc_id == del_doc_id) {
1252 
1253  ++*del_pos;
1254 
1255  /* Skip the entries for this document. */
1256  while (*enc->src_ilist_ptr) {
1257  fts_decode_vlc(&enc->src_ilist_ptr);
1258  }
1259 
1260  /* Skip the end of word position marker. */
1261  ++enc->src_ilist_ptr;
1262 
1263  } else {
1264 
1265  /* DOC ID already becomes larger than
1266  del_doc_id, check the next del_doc_id */
1267  if (del_doc_id > 0 && doc_id > del_doc_id) {
1268  del_doc_id = 0;
1269  ++*del_pos;
1270  delta = 0;
1271  goto test_again;
1272  }
1273 
1274  /* Decode and copy the word positions into
1275  the dest node. */
1276  fts_optimize_encode_node(dst_node, doc_id, enc);
1277 
1278  ++dst_node->doc_count;
1279 
1280  ut_a(dst_node->last_doc_id == doc_id);
1281  }
1282 
1283  /* Bytes copied so for from source. */
1284  copied = enc->src_ilist_ptr - src_node->ilist;
1285  }
1286 
1287  if (copied >= src_node->ilist_size) {
1288  ut_a(doc_id == src_node->last_doc_id);
1289  }
1290 
1291  enc->src_last_doc_id = doc_id;
1292 
1293  return(error);
1294 }
1295 
1296 /**********************************************************************/
1299 static __attribute__((nonnull, warn_unused_result))
1300 int
1301 fts_optimize_deleted_pos(
1302 /*=====================*/
1303  fts_optimize_t* optim,
1304  fts_word_t* word)
1305 {
1306  int del_pos;
1307  ib_vector_t* del_vec = optim->to_delete->doc_ids;
1308 
1309  /* Get the first and last dict ids for the word, we will use
1310  these values to determine which doc ids need to be removed
1311  when we coalesce the nodes. This way we can reduce the numer
1312  of elements that need to be searched in the deleted doc ids
1313  vector and secondly we can remove the doc ids during the
1314  coalescing phase. */
1315  if (ib_vector_size(del_vec) > 0) {
1316  fts_node_t* node;
1317  doc_id_t last_id;
1318  doc_id_t first_id;
1319  ulint size = ib_vector_size(word->nodes);
1320 
1321  node = (fts_node_t*) ib_vector_get(word->nodes, 0);
1322  first_id = node->first_doc_id;
1323 
1324  node = (fts_node_t*) ib_vector_get(word->nodes, size - 1);
1325  last_id = node->last_doc_id;
1326 
1327  ut_a(first_id <= last_id);
1328 
1329  del_pos = fts_optimize_lookup(
1330  del_vec, optim->del_pos, first_id, last_id);
1331  } else {
1332 
1333  del_pos = -1; /* Note that there is nothing to delete. */
1334  }
1335 
1336  return(del_pos);
1337 }
1338 
1339 #define FTS_DEBUG_PRINT
1340 /**********************************************************************/
1344 static
1345 ib_vector_t*
1346 fts_optimize_word(
1347 /*==============*/
1348  fts_optimize_t* optim,
1349  fts_word_t* word)
1350 {
1351  fts_encode_t enc;
1352  ib_vector_t* nodes;
1353  ulint i = 0;
1354  int del_pos;
1355  fts_node_t* dst_node = NULL;
1356  ib_vector_t* del_vec = optim->to_delete->doc_ids;
1357  ulint size = ib_vector_size(word->nodes);
1358 
1359  del_pos = fts_optimize_deleted_pos(optim, word);
1360  nodes = ib_vector_create(word->heap_alloc, sizeof(*dst_node), 128);
1361 
1362  enc.src_last_doc_id = 0;
1363  enc.src_ilist_ptr = NULL;
1364 
1365  if (fts_enable_diag_print) {
1366  word->text.f_str[word->text.f_len] = 0;
1367  fprintf(stderr, "FTS_OPTIMIZE: optimize \"%s\"\n",
1368  word->text.f_str);
1369  }
1370 
1371  while (i < size) {
1372  ulint copied;
1373  fts_node_t* src_node;
1374 
1375  src_node = (fts_node_t*) ib_vector_get(word->nodes, i);
1376 
1377  if (!dst_node) {
1378 
1379  dst_node = static_cast<fts_node_t*>(
1380  ib_vector_push(nodes, NULL));
1381  memset(dst_node, 0, sizeof(*dst_node));
1382  }
1383 
1384  /* Copy from the src to the dst node. */
1385  fts_optimize_node(del_vec, &del_pos, dst_node, src_node, &enc);
1386 
1387  ut_a(enc.src_ilist_ptr != NULL);
1388 
1389  /* Determine the numer of bytes copied to dst_node. */
1390  copied = enc.src_ilist_ptr - src_node->ilist;
1391 
1392  /* Can't copy more than whats in the vlc array. */
1393  ut_a(copied <= src_node->ilist_size);
1394 
1395  /* We are done with this node release the resources. */
1396  if (copied == src_node->ilist_size) {
1397 
1398  enc.src_last_doc_id = 0;
1399  enc.src_ilist_ptr = NULL;
1400 
1401  ut_free(src_node->ilist);
1402 
1403  src_node->ilist = NULL;
1404  src_node->ilist_size = src_node->ilist_size_alloc = 0;
1405 
1406  src_node = NULL;
1407 
1408  ++i; /* Get next source node to OPTIMIZE. */
1409  }
1410 
1411  if (dst_node->ilist_size >= FTS_ILIST_MAX_SIZE || i >= size) {
1412 
1413  dst_node = NULL;
1414  }
1415  }
1416 
1417  /* All dst nodes created should have been added to the vector. */
1418  ut_a(dst_node == NULL);
1419 
1420  /* Return the OPTIMIZED nodes. */
1421  return(nodes);
1422 }
1423 
1424 /**********************************************************************/
1427 static __attribute__((nonnull, warn_unused_result))
1428 dberr_t
1429 fts_optimize_write_word(
1430 /*====================*/
1431  trx_t* trx,
1432  fts_table_t* fts_table,
1433  fts_string_t* word,
1434  ib_vector_t* nodes)
1435 {
1436  ulint i;
1437  pars_info_t* info;
1438  que_t* graph;
1439  ulint selected;
1440  dberr_t error = DB_SUCCESS;
1441  char* table_name = fts_get_table_name(fts_table);
1442 
1443  info = pars_info_create();
1444 
1445  ut_ad(fts_table->charset);
1446 
1447  if (fts_enable_diag_print) {
1448  fprintf(stderr, "FTS_OPTIMIZE: processed \"%s\"\n",
1449  word->f_str);
1450  }
1451 
1452  pars_info_bind_varchar_literal(
1453  info, "word", word->f_str, word->f_len);
1454 
1455  selected = fts_select_index(fts_table->charset,
1456  word->f_str, word->f_len);
1457 
1458  fts_table->suffix = fts_get_suffix(selected);
1459 
1460  graph = fts_parse_sql(
1461  fts_table,
1462  info,
1463  "BEGIN DELETE FROM \"%s\" WHERE word = :word;");
1464 
1465  error = fts_eval_sql(trx, graph);
1466 
1467  if (error != DB_SUCCESS) {
1468  ut_print_timestamp(stderr);
1469  fprintf(stderr, " InnoDB: Error: (%s) during optimize, "
1470  "when deleting a word from the FTS index.\n",
1471  ut_strerr(error));
1472  }
1473 
1474  fts_que_graph_free(graph);
1475  graph = NULL;
1476 
1477  mem_free(table_name);
1478 
1479  /* Even if the operation needs to be rolled back and redone,
1480  we iterate over the nodes in order to free the ilist. */
1481  for (i = 0; i < ib_vector_size(nodes); ++i) {
1482 
1483  fts_node_t* node = (fts_node_t*) ib_vector_get(nodes, i);
1484 
1485  if (error == DB_SUCCESS) {
1486  error = fts_write_node(
1487  trx, &graph, fts_table, word, node);
1488 
1489  if (error != DB_SUCCESS) {
1490  ut_print_timestamp(stderr);
1491  fprintf(stderr, " InnoDB: Error: (%s) "
1492  "during optimize, while adding a "
1493  "word to the FTS index.\n",
1494  ut_strerr(error));
1495  }
1496  }
1497 
1498  ut_free(node->ilist);
1499  node->ilist = NULL;
1500  node->ilist_size = node->ilist_size_alloc = 0;
1501  }
1502 
1503  if (graph != NULL) {
1504  fts_que_graph_free(graph);
1505  }
1506 
1507  return(error);
1508 }
1509 
1510 /**********************************************************************/
1512 UNIV_INTERN
1513 void
1514 fts_word_free(
1515 /*==========*/
1516  fts_word_t* word)
1517 {
1518  mem_heap_t* heap = static_cast<mem_heap_t*>(word->heap_alloc->arg);
1519 
1520 #ifdef UNIV_DEBUG
1521  memset(word, 0, sizeof(*word));
1522 #endif /* UNIV_DEBUG */
1523 
1524  mem_heap_free(heap);
1525 }
1526 
1527 /**********************************************************************/
1530 static __attribute__((nonnull, warn_unused_result))
1531 dberr_t
1532 fts_optimize_compact(
1533 /*=================*/
1534  fts_optimize_t* optim,
1535  dict_index_t* index,
1536  ib_time_t start_time)
1537 {
1538  ulint i;
1539  dberr_t error = DB_SUCCESS;
1540  ulint size = ib_vector_size(optim->words);
1541 
1542  for (i = 0; i < size && error == DB_SUCCESS && !optim->done; ++i) {
1543  fts_word_t* word;
1544  ib_vector_t* nodes;
1545  trx_t* trx = optim->trx;
1546 
1547  word = (fts_word_t*) ib_vector_get(optim->words, i);
1548 
1549  /* nodes is allocated from the word heap and will be destroyed
1550  when the word is freed. We however have to be careful about
1551  the ilist, that needs to be freed explicitly. */
1552  nodes = fts_optimize_word(optim, word);
1553 
1554  /* Update the data on disk. */
1555  error = fts_optimize_write_word(
1556  trx, &optim->fts_index_table, &word->text, nodes);
1557 
1558  if (error == DB_SUCCESS) {
1559  /* Write the last word optimized to the config table,
1560  we use this value for restarting optimize. */
1561  error = fts_config_set_index_value(
1562  optim->trx, index,
1563  FTS_LAST_OPTIMIZED_WORD, &word->text);
1564  }
1565 
1566  /* Free the word that was optimized. */
1567  fts_word_free(word);
1568 
1569  if (fts_optimize_time_limit > 0
1570  && (ut_time() - start_time) > fts_optimize_time_limit) {
1571 
1572  optim->done = TRUE;
1573  }
1574  }
1575 
1576  return(error);
1577 }
1578 
1579 /**********************************************************************/
1582 static
1583 fts_optimize_t*
1584 fts_optimize_create(
1585 /*================*/
1586  dict_table_t* table)
1587 {
1588  fts_optimize_t* optim;
1589  mem_heap_t* heap = mem_heap_create(128);
1590 
1591  optim = (fts_optimize_t*) mem_heap_zalloc(heap, sizeof(*optim));
1592 
1593  optim->self_heap = ib_heap_allocator_create(heap);
1594 
1595  optim->to_delete = fts_doc_ids_create();
1596 
1597  optim->words = ib_vector_create(
1598  optim->self_heap, sizeof(fts_word_t), 256);
1599 
1600  optim->table = table;
1601 
1602  optim->trx = trx_allocate_for_background();
1603 
1604  optim->fts_common_table.parent = table->name;
1605  optim->fts_common_table.table_id = table->id;
1606  optim->fts_common_table.type = FTS_COMMON_TABLE;
1607 
1608  optim->fts_index_table.parent = table->name;
1609  optim->fts_index_table.table_id = table->id;
1610  optim->fts_index_table.type = FTS_INDEX_TABLE;
1611 
1612  /* The common prefix for all this parent table's aux tables. */
1613  optim->name_prefix = fts_get_table_name_prefix(
1614  &optim->fts_common_table);
1615 
1616  return(optim);
1617 }
1618 
1619 #ifdef FTS_OPTIMIZE_DEBUG
1620 /**********************************************************************/
1623 static __attribute__((nonnull, warn_unused_result))
1624 dberr_t
1625 fts_optimize_get_index_start_time(
1626 /*==============================*/
1627  trx_t* trx,
1628  dict_index_t* index,
1629  ib_time_t* start_time)
1630 {
1631  return(fts_config_get_index_ulint(
1632  trx, index, FTS_OPTIMIZE_START_TIME,
1633  (ulint*) start_time));
1634 }
1635 
1636 /**********************************************************************/
1639 static __attribute__((nonnull, warn_unused_result))
1640 dberr_t
1641 fts_optimize_set_index_start_time(
1642 /*==============================*/
1643  trx_t* trx,
1644  dict_index_t* index,
1645  ib_time_t start_time)
1646 {
1647  return(fts_config_set_index_ulint(
1648  trx, index, FTS_OPTIMIZE_START_TIME,
1649  (ulint) start_time));
1650 }
1651 
1652 /**********************************************************************/
1655 static __attribute__((nonnull, warn_unused_result))
1656 dberr_t
1657 fts_optimize_get_index_end_time(
1658 /*============================*/
1659  trx_t* trx,
1660  dict_index_t* index,
1661  ib_time_t* end_time)
1662 {
1663  return(fts_config_get_index_ulint(
1664  trx, index, FTS_OPTIMIZE_END_TIME, (ulint*) end_time));
1665 }
1666 
1667 /**********************************************************************/
1670 static __attribute__((nonnull, warn_unused_result))
1671 dberr_t
1672 fts_optimize_set_index_end_time(
1673 /*============================*/
1674  trx_t* trx,
1675  dict_index_t* index,
1676  ib_time_t end_time)
1677 {
1678  return(fts_config_set_index_ulint(
1679  trx, index, FTS_OPTIMIZE_END_TIME, (ulint) end_time));
1680 }
1681 #endif
1682 
1683 /**********************************************************************/
1685 static
1686 void
1687 fts_optimize_graph_free(
1688 /*====================*/
1689  fts_optimize_graph_t* graph)
1691 {
1692  if (graph->commit_graph) {
1693  que_graph_free(graph->commit_graph);
1694  graph->commit_graph = NULL;
1695  }
1696 
1697  if (graph->write_nodes_graph) {
1698  que_graph_free(graph->write_nodes_graph);
1699  graph->write_nodes_graph = NULL;
1700  }
1701 
1702  if (graph->delete_nodes_graph) {
1703  que_graph_free(graph->delete_nodes_graph);
1704  graph->delete_nodes_graph = NULL;
1705  }
1706 
1707  if (graph->read_nodes_graph) {
1708  que_graph_free(graph->read_nodes_graph);
1709  graph->read_nodes_graph = NULL;
1710  }
1711 }
1712 
1713 /**********************************************************************/
1715 static
1716 void
1717 fts_optimize_free(
1718 /*==============*/
1719  fts_optimize_t* optim)
1720 {
1721  mem_heap_t* heap = static_cast<mem_heap_t*>(optim->self_heap->arg);
1722 
1723  trx_free_for_background(optim->trx);
1724 
1725  fts_doc_ids_free(optim->to_delete);
1726  fts_optimize_graph_free(&optim->graph);
1727 
1728  mem_free(optim->name_prefix);
1729 
1730  /* This will free the heap from which optim itself was allocated. */
1731  mem_heap_free(heap);
1732 }
1733 
1734 /**********************************************************************/
1737 static
1738 ib_time_t
1739 fts_optimize_get_time_limit(
1740 /*========================*/
1741  trx_t* trx,
1742  fts_table_t* fts_table)
1743 {
1744  ib_time_t time_limit = 0;
1745 
1746  fts_config_get_ulint(
1747  trx, fts_table,
1748  FTS_OPTIMIZE_LIMIT_IN_SECS, (ulint*) &time_limit);
1749 
1750  return(time_limit * 1000);
1751 }
1752 
1753 
1754 /**********************************************************************/
1757 static
1758 void
1759 fts_optimize_words(
1760 /*===============*/
1761  fts_optimize_t* optim,
1762  dict_index_t* index,
1763  fts_string_t* word)
1764 {
1765  fts_fetch_t fetch;
1766  ib_time_t start_time;
1767  que_t* graph = NULL;
1768  CHARSET_INFO* charset = optim->fts_index_table.charset;
1769 
1770  ut_a(!optim->done);
1771 
1772  /* Get the time limit from the config table. */
1773  fts_optimize_time_limit = fts_optimize_get_time_limit(
1774  optim->trx, &optim->fts_common_table);
1775 
1776  start_time = ut_time();
1777 
1778  /* Setup the callback to use for fetching the word ilist etc. */
1779  fetch.read_arg = optim->words;
1780  fetch.read_record = fts_optimize_index_fetch_node;
1781 
1782  fprintf(stderr, "%.*s\n", (int) word->f_len, word->f_str);
1783 
1784  while(!optim->done) {
1785  dberr_t error;
1786  trx_t* trx = optim->trx;
1787  ulint selected;
1788 
1789  ut_a(ib_vector_size(optim->words) == 0);
1790 
1791  selected = fts_select_index(charset, word->f_str, word->f_len);
1792 
1793  /* Read the index records to optimize. */
1794  error = fts_index_fetch_nodes(
1795  trx, &graph, &optim->fts_index_table, word,
1796  &fetch);
1797 
1798  if (error == DB_SUCCESS) {
1799  /* There must be some nodes to read. */
1800  ut_a(ib_vector_size(optim->words) > 0);
1801 
1802  /* Optimize the nodes that were read and write
1803  back to DB. */
1804  error = fts_optimize_compact(optim, index, start_time);
1805 
1806  if (error == DB_SUCCESS) {
1807  fts_sql_commit(optim->trx);
1808  } else {
1809  fts_sql_rollback(optim->trx);
1810  }
1811  }
1812 
1813  ib_vector_reset(optim->words);
1814 
1815  if (error == DB_SUCCESS) {
1816  if (!optim->done) {
1817  if (!fts_zip_read_word(optim->zip, word)) {
1818  optim->done = TRUE;
1819  } else if (selected
1820  != fts_select_index(
1821  charset, word->f_str,
1822  word->f_len)
1823  && graph) {
1824  fts_que_graph_free(graph);
1825  graph = NULL;
1826  }
1827  }
1828  } else if (error == DB_LOCK_WAIT_TIMEOUT) {
1829  fprintf(stderr, "InnoDB: Warning: lock wait timeout "
1830  "during optimize. Retrying!\n");
1831 
1832  trx->error_state = DB_SUCCESS;
1833  } else if (error == DB_DEADLOCK) {
1834  fprintf(stderr, "InnoDB: Warning: deadlock "
1835  "during optimize. Retrying!\n");
1836 
1837  trx->error_state = DB_SUCCESS;
1838  } else {
1839  optim->done = TRUE; /* Exit the loop. */
1840  }
1841  }
1842 
1843  if (graph != NULL) {
1844  fts_que_graph_free(graph);
1845  }
1846 }
1847 
1848 /**********************************************************************/
1851 static
1852 ibool
1853 fts_optimize_set_next_word(
1854 /*=======================*/
1855  CHARSET_INFO* charset,
1856  fts_string_t* word)
1857 {
1858  ulint selected;
1859  ibool last = FALSE;
1860 
1861  selected = fts_select_next_index(charset, word->f_str, word->f_len);
1862 
1863  /* If this was the last index then reset to start. */
1864  if (fts_index_selector[selected].value == 0) {
1865  /* Reset the last optimized word to '' if no
1866  more words could be read from the FTS index. */
1867  word->f_len = 0;
1868  *word->f_str = 0;
1869 
1870  last = TRUE;
1871  } else {
1872  ulint value = fts_index_selector[selected].value;
1873 
1874  ut_a(value <= 0xff);
1875 
1876  /* Set to the first character of the next slot. */
1877  word->f_len = 1;
1878  *word->f_str = (byte) value;
1879  }
1880 
1881  return(last);
1882 }
1883 
1884 /**********************************************************************/
1888 static __attribute__((nonnull, warn_unused_result))
1889 dberr_t
1890 fts_optimize_index_completed(
1891 /*=========================*/
1892  fts_optimize_t* optim,
1893  dict_index_t* index)
1894 {
1895  fts_string_t word;
1896  dberr_t error;
1897  byte buf[sizeof(ulint)];
1898 #ifdef FTS_OPTIMIZE_DEBUG
1899  ib_time_t end_time = ut_time();
1900 
1901  error = fts_optimize_set_index_end_time(optim->trx, index, end_time);
1902 #endif
1903 
1904  /* If we've reached the end of the index then set the start
1905  word to the empty string. */
1906 
1907  word.f_len = 0;
1908  word.f_str = buf;
1909  *word.f_str = '\0';
1910 
1911  error = fts_config_set_index_value(
1912  optim->trx, index, FTS_LAST_OPTIMIZED_WORD, &word);
1913 
1914  if (error != DB_SUCCESS) {
1915 
1916  fprintf(stderr, "InnoDB: Error: (%s) while "
1917  "updating last optimized word!\n", ut_strerr(error));
1918  }
1919 
1920  return(error);
1921 }
1922 
1923 
1924 /**********************************************************************/
1928 static __attribute__((nonnull, warn_unused_result))
1929 dberr_t
1930 fts_optimize_index_read_words(
1931 /*==========================*/
1932  fts_optimize_t* optim,
1933  dict_index_t* index,
1934  fts_string_t* word)
1935 {
1936  dberr_t error = DB_SUCCESS;
1937 
1938  if (optim->del_list_regenerated) {
1939  word->f_len = 0;
1940  } else {
1941 
1942  /* Get the last word that was optimized from
1943  the config table. */
1944  error = fts_config_get_index_value(
1945  optim->trx, index, FTS_LAST_OPTIMIZED_WORD, word);
1946  }
1947 
1948  /* If record not found then we start from the top. */
1949  if (error == DB_RECORD_NOT_FOUND) {
1950  word->f_len = 0;
1951  error = DB_SUCCESS;
1952  }
1953 
1954  while (error == DB_SUCCESS) {
1955 
1956  error = fts_index_fetch_words(
1957  optim, word, fts_num_word_optimize);
1958 
1959  if (error == DB_SUCCESS) {
1960 
1961  /* If the search returned an empty set
1962  try the next index in the horizontal split. */
1963  if (optim->zip->n_words > 0) {
1964  break;
1965  } else {
1966 
1967  fts_optimize_set_next_word(
1968  optim->fts_index_table.charset,
1969  word);
1970 
1971  if (word->f_len == 0) {
1972  break;
1973  }
1974  }
1975  }
1976  }
1977 
1978  return(error);
1979 }
1980 
1981 /**********************************************************************/
1985 static __attribute__((nonnull, warn_unused_result))
1986 dberr_t
1987 fts_optimize_index(
1988 /*===============*/
1989  fts_optimize_t* optim,
1990  dict_index_t* index)
1991 {
1992  fts_string_t word;
1993  dberr_t error;
1994  byte str[FTS_MAX_WORD_LEN + 1];
1995 
1996  /* Set the current index that we have to optimize. */
1997  optim->fts_index_table.index_id = index->id;
1998  optim->fts_index_table.charset = fts_index_get_charset(index);
1999 
2000  optim->done = FALSE; /* Optimize until !done */
2001 
2002  /* We need to read the last word optimized so that we start from
2003  the next word. */
2004  word.f_str = str;
2005 
2006  /* We set the length of word to the size of str since we
2007  need to pass the max len info to the fts_get_config_value() function. */
2008  word.f_len = sizeof(str) - 1;
2009 
2010  memset(word.f_str, 0x0, word.f_len);
2011 
2012  /* Read the words that will be optimized in this pass. */
2013  error = fts_optimize_index_read_words(optim, index, &word);
2014 
2015  if (error == DB_SUCCESS) {
2016  int zip_error;
2017 
2018  ut_a(optim->zip->pos == 0);
2019  ut_a(optim->zip->zp->total_in == 0);
2020  ut_a(optim->zip->zp->total_out == 0);
2021 
2022  zip_error = inflateInit(optim->zip->zp);
2023  ut_a(zip_error == Z_OK);
2024 
2025  word.f_len = 0;
2026  word.f_str = str;
2027 
2028  /* Read the first word to optimize from the Zip buffer. */
2029  if (!fts_zip_read_word(optim->zip, &word)) {
2030 
2031  optim->done = TRUE;
2032  } else {
2033  fts_optimize_words(optim, index, &word);
2034  }
2035 
2036  /* If we couldn't read any records then optimize is
2037  complete. Increment the number of indexes that have
2038  been optimized and set FTS index optimize state to
2039  completed. */
2040  if (error == DB_SUCCESS && optim->zip->n_words == 0) {
2041 
2042  error = fts_optimize_index_completed(optim, index);
2043 
2044  if (error == DB_SUCCESS) {
2045  ++optim->n_completed;
2046  }
2047  }
2048  }
2049 
2050  return(error);
2051 }
2052 
2053 /**********************************************************************/
2056 static __attribute__((nonnull, warn_unused_result))
2057 dberr_t
2058 fts_optimize_purge_deleted_doc_ids(
2059 /*===============================*/
2060  fts_optimize_t* optim)
2061 {
2062  ulint i;
2063  pars_info_t* info;
2064  que_t* graph;
2065  fts_update_t* update;
2066  char* sql_str;
2067  doc_id_t write_doc_id;
2068  dberr_t error = DB_SUCCESS;
2069 
2070  info = pars_info_create();
2071 
2072  ut_a(ib_vector_size(optim->to_delete->doc_ids) > 0);
2073 
2074  update = static_cast<fts_update_t*>(
2075  ib_vector_get(optim->to_delete->doc_ids, 0));
2076 
2077  /* Convert to "storage" byte order. */
2078  fts_write_doc_id((byte*) &write_doc_id, update->doc_id);
2079 
2080  /* This is required for the SQL parser to work. It must be able
2081  to find the following variables. So we do it twice. */
2082  fts_bind_doc_id(info, "doc_id1", &write_doc_id);
2083  fts_bind_doc_id(info, "doc_id2", &write_doc_id);
2084 
2085  /* Since we only replace the table_id and don't construct the full
2086  name, we do substitution ourselves. Remember to free sql_str. */
2087  sql_str = ut_strreplace(
2088  fts_delete_doc_ids_sql, "%s", optim->name_prefix);
2089 
2090  graph = fts_parse_sql(NULL, info, sql_str);
2091 
2092  mem_free(sql_str);
2093 
2094  /* Delete the doc ids that were copied at the start. */
2095  for (i = 0; i < ib_vector_size(optim->to_delete->doc_ids); ++i) {
2096 
2097  update = static_cast<fts_update_t*>(ib_vector_get(
2098  optim->to_delete->doc_ids, i));
2099 
2100  /* Convert to "storage" byte order. */
2101  fts_write_doc_id((byte*) &write_doc_id, update->doc_id);
2102 
2103  fts_bind_doc_id(info, "doc_id1", &write_doc_id);
2104 
2105  fts_bind_doc_id(info, "doc_id2", &write_doc_id);
2106 
2107  error = fts_eval_sql(optim->trx, graph);
2108 
2109  // FIXME: Check whether delete actually succeeded!
2110  if (error != DB_SUCCESS) {
2111 
2112  fts_sql_rollback(optim->trx);
2113  break;
2114  }
2115  }
2116 
2117  fts_que_graph_free(graph);
2118 
2119  return(error);
2120 }
2121 
2122 /**********************************************************************/
2125 static __attribute__((nonnull, warn_unused_result))
2126 dberr_t
2127 fts_optimize_purge_deleted_doc_id_snapshot(
2128 /*=======================================*/
2129  fts_optimize_t* optim)
2130 {
2131  dberr_t error;
2132  que_t* graph;
2133  char* sql_str;
2134 
2135  /* Since we only replace the table_id and don't construct
2136  the full name, we do the '%s' substitution ourselves. */
2137  sql_str = ut_strreplace(fts_end_delete_sql, "%s", optim->name_prefix);
2138 
2139  /* Delete the doc ids that were copied to delete pending state at
2140  the start of optimize. */
2141  graph = fts_parse_sql(NULL, NULL, sql_str);
2142 
2143  mem_free(sql_str);
2144 
2145  error = fts_eval_sql(optim->trx, graph);
2146  fts_que_graph_free(graph);
2147 
2148  return(error);
2149 }
2150 
2151 /**********************************************************************/
2156 static
2157 ulint
2158 fts_optimize_being_deleted_count(
2159 /*=============================*/
2160  fts_optimize_t* optim)
2161 {
2162  fts_table_t fts_table;
2163 
2164  FTS_INIT_FTS_TABLE(&fts_table, "BEING_DELETED", FTS_COMMON_TABLE,
2165  optim->table);
2166 
2167  return(fts_get_rows_count(&fts_table));
2168 }
2169 
2170 /*********************************************************************/
2175 static __attribute__((nonnull, warn_unused_result))
2176 dberr_t
2177 fts_optimize_create_deleted_doc_id_snapshot(
2178 /*========================================*/
2179  fts_optimize_t* optim)
2180 {
2181  dberr_t error;
2182  que_t* graph;
2183  char* sql_str;
2184 
2185  /* Since we only replace the table_id and don't construct the
2186  full name, we do the substitution ourselves. */
2187  sql_str = ut_strreplace(fts_init_delete_sql, "%s", optim->name_prefix);
2188 
2189  /* Move doc_ids that are to be deleted to state being deleted. */
2190  graph = fts_parse_sql(NULL, NULL, sql_str);
2191 
2192  mem_free(sql_str);
2193 
2194  error = fts_eval_sql(optim->trx, graph);
2195 
2196  fts_que_graph_free(graph);
2197 
2198  if (error != DB_SUCCESS) {
2199  fts_sql_rollback(optim->trx);
2200  } else {
2201  fts_sql_commit(optim->trx);
2202  }
2203 
2204  optim->del_list_regenerated = TRUE;
2205 
2206  return(error);
2207 }
2208 
2209 /*********************************************************************/
2213 static __attribute__((nonnull, warn_unused_result))
2214 dberr_t
2215 fts_optimize_read_deleted_doc_id_snapshot(
2216 /*======================================*/
2217  fts_optimize_t* optim)
2218 {
2219  dberr_t error;
2220 
2221  optim->fts_common_table.suffix = "BEING_DELETED";
2222 
2223  /* Read the doc_ids to delete. */
2224  error = fts_table_fetch_doc_ids(
2225  optim->trx, &optim->fts_common_table, optim->to_delete);
2226 
2227  if (error == DB_SUCCESS) {
2228 
2229  optim->fts_common_table.suffix = "BEING_DELETED_CACHE";
2230 
2231  /* Read additional doc_ids to delete. */
2232  error = fts_table_fetch_doc_ids(
2233  optim->trx, &optim->fts_common_table, optim->to_delete);
2234  }
2235 
2236  if (error != DB_SUCCESS) {
2237 
2238  fts_doc_ids_free(optim->to_delete);
2239  optim->to_delete = NULL;
2240  }
2241 
2242  return(error);
2243 }
2244 
2245 /*********************************************************************/
2250 static __attribute__((nonnull, warn_unused_result))
2251 dberr_t
2252 fts_optimize_indexes(
2253 /*=================*/
2254  fts_optimize_t* optim)
2255 {
2256  ulint i;
2257  dberr_t error = DB_SUCCESS;
2258  fts_t* fts = optim->table->fts;
2259 
2260  /* Optimize the FTS indexes. */
2261  for (i = 0; i < ib_vector_size(fts->indexes); ++i) {
2262  dict_index_t* index;
2263 
2264 #ifdef FTS_OPTIMIZE_DEBUG
2265  ib_time_t end_time;
2266  ib_time_t start_time;
2267 
2268  /* Get the start and end optimize times for this index. */
2269  error = fts_optimize_get_index_start_time(
2270  optim->trx, index, &start_time);
2271 
2272  if (error != DB_SUCCESS) {
2273  break;
2274  }
2275 
2276  error = fts_optimize_get_index_end_time(
2277  optim->trx, index, &end_time);
2278 
2279  if (error != DB_SUCCESS) {
2280  break;
2281  }
2282 
2283  /* Start time will be 0 only for the first time or after
2284  completing the optimization of all FTS indexes. */
2285  if (start_time == 0) {
2286  start_time = ut_time();
2287 
2288  error = fts_optimize_set_index_start_time(
2289  optim->trx, index, start_time);
2290  }
2291 
2292  /* Check if this index needs to be optimized or not. */
2293  if (ut_difftime(end_time, start_time) < 0) {
2294  error = fts_optimize_index(optim, index);
2295 
2296  if (error != DB_SUCCESS) {
2297  break;
2298  }
2299  } else {
2300  ++optim->n_completed;
2301  }
2302 #endif
2303  index = static_cast<dict_index_t*>(
2304  ib_vector_getp(fts->indexes, i));
2305  error = fts_optimize_index(optim, index);
2306  }
2307 
2308  if (error == DB_SUCCESS) {
2309  fts_sql_commit(optim->trx);
2310  } else {
2311  fts_sql_rollback(optim->trx);
2312  }
2313 
2314  return(error);
2315 }
2316 
2317 /*********************************************************************/
2320 static __attribute__((nonnull, warn_unused_result))
2321 dberr_t
2322 fts_optimize_purge_snapshot(
2323 /*========================*/
2324  fts_optimize_t* optim)
2325 {
2326  dberr_t error;
2327 
2328  /* Delete the doc ids from the master deleted tables, that were
2329  in the snapshot that was taken at the start of optimize. */
2330  error = fts_optimize_purge_deleted_doc_ids(optim);
2331 
2332  if (error == DB_SUCCESS) {
2333  /* Destroy the deleted doc id snapshot. */
2334  error = fts_optimize_purge_deleted_doc_id_snapshot(optim);
2335  }
2336 
2337  if (error == DB_SUCCESS) {
2338  fts_sql_commit(optim->trx);
2339  } else {
2340  fts_sql_rollback(optim->trx);
2341  }
2342 
2343  return(error);
2344 }
2345 
2346 /*********************************************************************/
2349 static __attribute__((nonnull, warn_unused_result))
2350 dberr_t
2351 fts_optimize_reset_start_time(
2352 /*==========================*/
2353  fts_optimize_t* optim)
2354 {
2355  dberr_t error = DB_SUCCESS;
2356 #ifdef FTS_OPTIMIZE_DEBUG
2357  fts_t* fts = optim->table->fts;
2358 
2359  /* Optimization should have been completed for all indexes. */
2360  ut_a(optim->n_completed == ib_vector_size(fts->indexes));
2361 
2362  for (uint i = 0; i < ib_vector_size(fts->indexes); ++i) {
2363  dict_index_t* index;
2364 
2365  ib_time_t start_time = 0;
2366 
2367  /* Reset the start time to 0 for this index. */
2368  error = fts_optimize_set_index_start_time(
2369  optim->trx, index, start_time);
2370 
2371  index = static_cast<dict_index_t*>(
2372  ib_vector_getp(fts->indexes, i));
2373  }
2374 #endif
2375 
2376  if (error == DB_SUCCESS) {
2377  fts_sql_commit(optim->trx);
2378  } else {
2379  fts_sql_rollback(optim->trx);
2380  }
2381 
2382  return(error);
2383 }
2384 
2385 /*********************************************************************/
2388 static __attribute__((nonnull))
2389 dberr_t
2390 fts_optimize_table_bk(
2391 /*==================*/
2392  fts_slot_t* slot)
2393 {
2394  dberr_t error;
2395  dict_table_t* table = slot->table;
2396  fts_t* fts = table->fts;
2397 
2398  /* Avoid optimizing tables that were optimized recently. */
2399  if (slot->last_run > 0
2400  && (ut_time() - slot->last_run) < slot->interval_time) {
2401 
2402  return(DB_SUCCESS);
2403 
2404  } else if (fts && fts->cache
2405  && fts->cache->deleted >= FTS_OPTIMIZE_THRESHOLD) {
2406 
2407  error = fts_optimize_table(table);
2408 
2409  if (error == DB_SUCCESS) {
2410  slot->state = FTS_STATE_DONE;
2411  slot->last_run = 0;
2412  slot->completed = ut_time();
2413  }
2414  } else {
2415  error = DB_SUCCESS;
2416  }
2417 
2418  /* Note time this run completed. */
2419  slot->last_run = ut_time();
2420 
2421  return(error);
2422 }
2423 /*********************************************************************/
2426 UNIV_INTERN
2427 dberr_t
2428 fts_optimize_table(
2429 /*===============*/
2430  dict_table_t* table)
2431 {
2432  dberr_t error = DB_SUCCESS;
2433  fts_optimize_t* optim = NULL;
2434  fts_t* fts = table->fts;
2435 
2436  ut_print_timestamp(stderr);
2437  fprintf(stderr, " InnoDB: FTS start optimize %s\n", table->name);
2438 
2439  optim = fts_optimize_create(table);
2440 
2441  // FIXME: Call this only at the start of optimize, currently we
2442  // rely on DB_DUPLICATE_KEY to handle corrupting the snapshot.
2443 
2444  /* Check whether there are still records in BEING_DELETED table */
2445  if (fts_optimize_being_deleted_count(optim) == 0) {
2446  /* Take a snapshot of the deleted document ids, they are copied
2447  to the BEING_ tables. */
2448  error = fts_optimize_create_deleted_doc_id_snapshot(optim);
2449  }
2450 
2451  /* A duplicate error is OK, since we don't erase the
2452  doc ids from the being deleted state until all FTS
2453  indexes have been optimized. */
2454  if (error == DB_DUPLICATE_KEY) {
2455  error = DB_SUCCESS;
2456  }
2457 
2458  if (error == DB_SUCCESS) {
2459 
2460  /* These document ids will be filtered out during the
2461  index optimization phase. They are in the snapshot that we
2462  took above, at the start of the optimize. */
2463  error = fts_optimize_read_deleted_doc_id_snapshot(optim);
2464 
2465  if (error == DB_SUCCESS) {
2466 
2467  /* Commit the read of being deleted
2468  doc ids transaction. */
2469  fts_sql_commit(optim->trx);
2470 
2471  /* We would do optimization only if there
2472  are deleted records to be cleaned up */
2473  if (ib_vector_size(optim->to_delete->doc_ids) > 0) {
2474  error = fts_optimize_indexes(optim);
2475  }
2476 
2477  } else {
2478  ut_a(optim->to_delete == NULL);
2479  }
2480 
2481  /* Only after all indexes have been optimized can we
2482  delete the (snapshot) doc ids in the pending delete,
2483  and master deleted tables. */
2484  if (error == DB_SUCCESS
2485  && optim->n_completed == ib_vector_size(fts->indexes)) {
2486 
2487  if (fts_enable_diag_print) {
2488  fprintf(stderr, "FTS_OPTIMIZE: Completed "
2489  "Optimize, cleanup DELETED "
2490  "table\n");
2491  }
2492 
2493  if (ib_vector_size(optim->to_delete->doc_ids) > 0) {
2494 
2495  /* Purge the doc ids that were in the
2496  snapshot from the snapshot tables and
2497  the master deleted table. */
2498  error = fts_optimize_purge_snapshot(optim);
2499  }
2500 
2501  if (error == DB_SUCCESS) {
2502  /* Reset the start time of all the FTS indexes
2503  so that optimize can be restarted. */
2504  error = fts_optimize_reset_start_time(optim);
2505  }
2506  }
2507  }
2508 
2509  fts_optimize_free(optim);
2510 
2511  ut_print_timestamp(stderr);
2512  fprintf(stderr, " InnoDB: FTS end optimize %s\n", table->name);
2513 
2514  return(error);
2515 }
2516 
2517 /********************************************************************/
2520 static
2521 fts_msg_t*
2522 fts_optimize_create_msg(
2523 /*====================*/
2524  fts_msg_type_t type,
2525  void* ptr)
2526 {
2527  mem_heap_t* heap;
2528  fts_msg_t* msg;
2529 
2530  heap = mem_heap_create(sizeof(*msg) + sizeof(ib_list_node_t) + 16);
2531  msg = static_cast<fts_msg_t*>(mem_heap_alloc(heap, sizeof(*msg)));
2532 
2533  msg->ptr = ptr;
2534  msg->type = type;
2535  msg->heap = heap;
2536 
2537  return(msg);
2538 }
2539 
2540 /**********************************************************************/
2542 UNIV_INTERN
2543 void
2544 fts_optimize_add_table(
2545 /*===================*/
2546  dict_table_t* table)
2547 {
2548  fts_msg_t* msg;
2549 
2550  if (!fts_optimize_wq) {
2551  return;
2552  }
2553 
2554  /* Make sure table with FTS index cannot be evicted */
2555  if (table->can_be_evicted) {
2556  dict_table_move_from_lru_to_non_lru(table);
2557  }
2558 
2559  msg = fts_optimize_create_msg(FTS_MSG_ADD_TABLE, table);
2560 
2561  ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
2562 }
2563 
2564 /**********************************************************************/
2566 UNIV_INTERN
2567 void
2568 fts_optimize_do_table(
2569 /*==================*/
2570  dict_table_t* table)
2571 {
2572  fts_msg_t* msg;
2573 
2574  /* Optimizer thread could be shutdown */
2575  if (!fts_optimize_wq) {
2576  return;
2577  }
2578 
2579  msg = fts_optimize_create_msg(FTS_MSG_OPTIMIZE_TABLE, table);
2580 
2581  ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
2582 }
2583 
2584 /**********************************************************************/
2587 UNIV_INTERN
2588 void
2589 fts_optimize_remove_table(
2590 /*======================*/
2591  dict_table_t* table)
2592 {
2593  fts_msg_t* msg;
2594  os_event_t event;
2595  fts_msg_del_t* remove;
2596 
2597  /* if the optimize system not yet initialized, return */
2598  if (!fts_optimize_wq) {
2599  return;
2600  }
2601 
2602  /* FTS optimizer thread is already exited */
2603  if (fts_opt_start_shutdown) {
2604  ib_logf(IB_LOG_LEVEL_INFO,
2605  "Try to remove table %s after FTS optimize"
2606  " thread exiting.", table->name);
2607  return;
2608  }
2609 
2610  msg = fts_optimize_create_msg(FTS_MSG_DEL_TABLE, NULL);
2611 
2612  /* We will wait on this event until signalled by the consumer. */
2613  event = os_event_create();
2614 
2615  remove = static_cast<fts_msg_del_t*>(
2616  mem_heap_alloc(msg->heap, sizeof(*remove)));
2617 
2618  remove->table = table;
2619  remove->event = event;
2620  msg->ptr = remove;
2621 
2622  ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
2623 
2624  os_event_wait(event);
2625 
2626  os_event_free(event);
2627 }
2628 
2629 /**********************************************************************/
2632 static
2633 fts_slot_t*
2634 fts_optimize_find_slot(
2635 /*===================*/
2636  ib_vector_t* tables,
2637  const dict_table_t* table)
2638 {
2639  ulint i;
2640 
2641  for (i = 0; i < ib_vector_size(tables); ++i) {
2642  fts_slot_t* slot;
2643 
2644  slot = static_cast<fts_slot_t*>(ib_vector_get(tables, i));
2645 
2646  if (slot->table->id == table->id) {
2647  return(slot);
2648  }
2649  }
2650 
2651  return(NULL);
2652 }
2653 
2654 /**********************************************************************/
2656 static
2657 void
2658 fts_optimize_start_table(
2659 /*=====================*/
2660  ib_vector_t* tables,
2661  dict_table_t* table)
2662 {
2663  fts_slot_t* slot;
2664 
2665  slot = fts_optimize_find_slot(tables, table);
2666 
2667  if (slot == NULL) {
2668  ut_print_timestamp(stderr);
2669  fprintf(stderr, " InnoDB: Error: table %s not registered "
2670  "with the optimize thread.\n", table->name);
2671  } else {
2672  slot->last_run = 0;
2673  slot->completed = 0;
2674  }
2675 }
2676 
2677 /**********************************************************************/
2679 static
2680 ibool
2681 fts_optimize_new_table(
2682 /*===================*/
2683  ib_vector_t* tables,
2684  dict_table_t* table)
2685 {
2686  ulint i;
2687  fts_slot_t* slot;
2688  ulint empty_slot = ULINT_UNDEFINED;
2689 
2690  /* Search for duplicates, also find a free slot if one exists. */
2691  for (i = 0; i < ib_vector_size(tables); ++i) {
2692 
2693  slot = static_cast<fts_slot_t*>(
2694  ib_vector_get(tables, i));
2695 
2696  if (slot->state == FTS_STATE_EMPTY) {
2697  empty_slot = i;
2698  } else if (slot->table->id == table->id) {
2699  /* Already exists in our optimize queue. */
2700  return(FALSE);
2701  }
2702  }
2703 
2704  /* Reuse old slot. */
2705  if (empty_slot != ULINT_UNDEFINED) {
2706 
2707  slot = static_cast<fts_slot_t*>(
2708  ib_vector_get(tables, empty_slot));
2709 
2710  ut_a(slot->state == FTS_STATE_EMPTY);
2711 
2712  } else { /* Create a new slot. */
2713 
2714  slot = static_cast<fts_slot_t*>(ib_vector_push(tables, NULL));
2715  }
2716 
2717  memset(slot, 0x0, sizeof(*slot));
2718 
2719  slot->table = table;
2720  slot->state = FTS_STATE_LOADED;
2721  slot->interval_time = FTS_OPTIMIZE_INTERVAL_IN_SECS;
2722 
2723  return(TRUE);
2724 }
2725 
2726 /**********************************************************************/
2728 static
2729 ibool
2730 fts_optimize_del_table(
2731 /*===================*/
2732  ib_vector_t* tables,
2733  fts_msg_del_t* msg)
2734 {
2735  ulint i;
2736  dict_table_t* table = msg->table;
2737 
2738  for (i = 0; i < ib_vector_size(tables); ++i) {
2739  fts_slot_t* slot;
2740 
2741  slot = static_cast<fts_slot_t*>(ib_vector_get(tables, i));
2742 
2743  /* FIXME: Should we assert on this ? */
2744  if (slot->state != FTS_STATE_EMPTY
2745  && slot->table->id == table->id) {
2746 
2747  ut_print_timestamp(stderr);
2748  fprintf(stderr, " InnoDB: FTS Optimize Removing "
2749  "table %s\n", table->name);
2750 
2751  slot->table = NULL;
2752  slot->state = FTS_STATE_EMPTY;
2753 
2754  return(TRUE);
2755  }
2756  }
2757 
2758  return(FALSE);
2759 }
2760 
2761 /**********************************************************************/
2764 static
2765 ulint
2766 fts_optimize_how_many(
2767 /*==================*/
2768  const ib_vector_t* tables)
2770 {
2771  ulint i;
2772  ib_time_t delta;
2773  ulint n_tables = 0;
2774  ib_time_t current_time;
2775 
2776  current_time = ut_time();
2777 
2778  for (i = 0; i < ib_vector_size(tables); ++i) {
2779  const fts_slot_t* slot;
2780 
2781  slot = static_cast<const fts_slot_t*>(
2782  ib_vector_get_const(tables, i));
2783 
2784  switch (slot->state) {
2785  case FTS_STATE_DONE:
2786  case FTS_STATE_LOADED:
2787  ut_a(slot->completed <= current_time);
2788 
2789  delta = current_time - slot->completed;
2790 
2791  /* Skip slots that have been optimized recently. */
2792  if (delta >= slot->interval_time) {
2793  ++n_tables;
2794  }
2795  break;
2796 
2797  case FTS_STATE_RUNNING:
2798  ut_a(slot->last_run <= current_time);
2799 
2800  delta = current_time - slot->last_run;
2801 
2802  if (delta > slot->interval_time) {
2803  ++n_tables;
2804  }
2805  break;
2806 
2807  /* Slots in a state other than the above
2808  are ignored. */
2809  case FTS_STATE_EMPTY:
2810  case FTS_STATE_SUSPENDED:
2811  break;
2812  }
2813 
2814  }
2815 
2816  return(n_tables);
2817 }
2818 
2819 /**********************************************************************/
2822 static
2823 bool
2824 fts_is_sync_needed(
2825 /*===============*/
2826  const ib_vector_t* tables)
2828 {
2829  ulint total_memory = 0;
2830  double time_diff = difftime(ut_time(), last_check_sync_time);
2831 
2832  if (fts_need_sync || time_diff < 5) {
2833  return(false);
2834  }
2835 
2836  last_check_sync_time = ut_time();
2837 
2838  for (ulint i = 0; i < ib_vector_size(tables); ++i) {
2839  const fts_slot_t* slot;
2840 
2841  slot = static_cast<const fts_slot_t*>(
2842  ib_vector_get_const(tables, i));
2843 
2844  if (slot->table && slot->table->fts) {
2845  total_memory += slot->table->fts->cache->total_size;
2846  }
2847 
2848  if (total_memory > fts_max_total_cache_size) {
2849  return(true);
2850  }
2851  }
2852 
2853  return(false);
2854 }
2855 
2856 #if 0
2857 /*********************************************************************/
2859 static
2860 void
2861 fts_optimize_need_sync(
2862 /*===================*/
2863  ib_vector_t* tables)
2864 {
2865  dict_table_t* table = NULL;
2866  fts_slot_t* slot;
2867  ulint num_table = ib_vector_size(tables);
2868 
2869  if (!num_table) {
2870  return;
2871  }
2872 
2873  if (fts_optimize_sync_iterator >= num_table) {
2874  fts_optimize_sync_iterator = 0;
2875  }
2876 
2877  slot = ib_vector_get(tables, fts_optimize_sync_iterator);
2878  table = slot->table;
2879 
2880  if (!table) {
2881  return;
2882  }
2883 
2884  ut_ad(table->fts);
2885 
2886  if (table->fts->cache) {
2887  ulint deleted = table->fts->cache->deleted;
2888 
2889  if (table->fts->cache->added
2890  >= fts_optimize_add_threshold) {
2891  fts_sync_table(table);
2892  } else if (deleted >= fts_optimize_delete_threshold) {
2893  fts_optimize_do_table(table);
2894 
2895  mutex_enter(&table->fts->cache->deleted_lock);
2896  table->fts->cache->deleted -= deleted;
2897  mutex_exit(&table->fts->cache->deleted_lock);
2898  }
2899  }
2900 
2901  fts_optimize_sync_iterator++;
2902 
2903  return;
2904 }
2905 #endif
2906 
2907 /**********************************************************************/
2910 UNIV_INTERN
2911 os_thread_ret_t
2912 fts_optimize_thread(
2913 /*================*/
2914  void* arg)
2915 {
2916  mem_heap_t* heap;
2917  ib_vector_t* tables;
2918  ib_alloc_t* heap_alloc;
2919  ulint current = 0;
2920  ibool done = FALSE;
2921  ulint n_tables = 0;
2922  os_event_t exit_event = 0;
2923  ulint n_optimize = 0;
2924  ib_wqueue_t* wq = (ib_wqueue_t*) arg;
2925 
2926  ut_ad(!srv_read_only_mode);
2927 
2928  heap = mem_heap_create(sizeof(dict_table_t*) * 64);
2929  heap_alloc = ib_heap_allocator_create(heap);
2930 
2931  tables = ib_vector_create(heap_alloc, sizeof(fts_slot_t), 4);
2932 
2933  while(!done && srv_shutdown_state == SRV_SHUTDOWN_NONE) {
2934 
2935  /* If there is no message in the queue and we have tables
2936  to optimize then optimize the tables. */
2937 
2938  if (!done
2939  && ib_wqueue_is_empty(wq)
2940  && n_tables > 0
2941  && n_optimize > 0) {
2942 
2943  fts_slot_t* slot;
2944 
2945  ut_a(ib_vector_size(tables) > 0);
2946 
2947  slot = static_cast<fts_slot_t*>(
2948  ib_vector_get(tables, current));
2949 
2950  /* Handle the case of empty slots. */
2951  if (slot->state != FTS_STATE_EMPTY) {
2952 
2953  slot->state = FTS_STATE_RUNNING;
2954 
2955  fts_optimize_table_bk(slot);
2956  }
2957 
2958  ++current;
2959 
2960  /* Wrap around the counter. */
2961  if (current >= ib_vector_size(tables)) {
2962  n_optimize = fts_optimize_how_many(tables);
2963 
2964  current = 0;
2965  }
2966 
2967  } else if (n_optimize == 0 || !ib_wqueue_is_empty(wq)) {
2968  fts_msg_t* msg;
2969 
2970  msg = static_cast<fts_msg_t*>(
2971  ib_wqueue_timedwait(wq,
2972  FTS_QUEUE_WAIT_IN_USECS));
2973 
2974  /* Timeout ? */
2975  if (msg == NULL) {
2976  if (fts_is_sync_needed(tables)) {
2977  fts_need_sync = true;
2978  }
2979 
2980  continue;
2981  }
2982 
2983  switch (msg->type) {
2984  case FTS_MSG_START:
2985  break;
2986 
2987  case FTS_MSG_PAUSE:
2988  break;
2989 
2990  case FTS_MSG_STOP:
2991  done = TRUE;
2992  exit_event = (os_event_t) msg->ptr;
2993  break;
2994 
2995  case FTS_MSG_ADD_TABLE:
2996  ut_a(!done);
2997  if (fts_optimize_new_table(
2998  tables,
2999  static_cast<dict_table_t*>(
3000  msg->ptr))) {
3001  ++n_tables;
3002  }
3003  break;
3004 
3005  case FTS_MSG_OPTIMIZE_TABLE:
3006  if (!done) {
3007  fts_optimize_start_table(
3008  tables,
3009  static_cast<dict_table_t*>(
3010  msg->ptr));
3011  }
3012  break;
3013 
3014  case FTS_MSG_DEL_TABLE:
3015  if (fts_optimize_del_table(
3016  tables, static_cast<fts_msg_del_t*>(
3017  msg->ptr))) {
3018  --n_tables;
3019  }
3020 
3021  /* Signal the producer that we have
3022  removed the table. */
3023  os_event_set(
3024  ((fts_msg_del_t*) msg->ptr)->event);
3025  break;
3026 
3027  default:
3028  ut_error;
3029  }
3030 
3031  mem_heap_free(msg->heap);
3032 
3033  if (!done) {
3034  n_optimize = fts_optimize_how_many(tables);
3035  } else {
3036  n_optimize = 0;
3037  }
3038  }
3039  }
3040 
3041  /* Server is being shutdown, sync the data from FTS cache to disk
3042  if needed */
3043  if (n_tables > 0) {
3044  ulint i;
3045 
3046  for (i = 0; i < ib_vector_size(tables); i++) {
3047  fts_slot_t* slot;
3048 
3049  slot = static_cast<fts_slot_t*>(
3050  ib_vector_get(tables, i));
3051 
3052  if (slot->state != FTS_STATE_EMPTY) {
3053  dict_table_t* table = NULL;
3054 
3055  table = dict_table_open_on_name(
3056  slot->table->name, FALSE, FALSE,
3057  DICT_ERR_IGNORE_INDEX_ROOT);
3058 
3059  if (table) {
3060 
3061  if (dict_table_has_fts_index(table)) {
3062  fts_sync_table(table);
3063  }
3064 
3065  if (table->fts) {
3066  fts_free(table);
3067  }
3068 
3069  dict_table_close(table, FALSE, FALSE);
3070  }
3071  }
3072  }
3073  }
3074 
3075  ib_vector_free(tables);
3076 
3077  ib_logf(IB_LOG_LEVEL_INFO, "FTS optimize thread exiting.");
3078 
3079  os_event_set(exit_event);
3080 
3081  /* We count the number of threads in os_thread_exit(). A created
3082  thread should always use that to exit and not use return() to exit. */
3083  os_thread_exit(NULL);
3084 
3085  OS_THREAD_DUMMY_RETURN;
3086 }
3087 
3088 /**********************************************************************/
3090 UNIV_INTERN
3091 void
3092 fts_optimize_init(void)
3093 /*===================*/
3094 {
3095  ut_ad(!srv_read_only_mode);
3096 
3097  /* For now we only support one optimize thread. */
3098  ut_a(fts_optimize_wq == NULL);
3099 
3100  fts_optimize_wq = ib_wqueue_create();
3101  ut_a(fts_optimize_wq != NULL);
3102  last_check_sync_time = ut_time();
3103 
3104  os_thread_create(fts_optimize_thread, fts_optimize_wq, NULL);
3105 }
3106 
3107 /**********************************************************************/
3110 UNIV_INTERN
3111 ibool
3112 fts_optimize_is_init(void)
3113 /*======================*/
3114 {
3115  return(fts_optimize_wq != NULL);
3116 }
3117 
3118 /**********************************************************************/
3120 UNIV_INTERN
3121 void
3122 fts_optimize_start_shutdown(void)
3123 /*=============================*/
3124 {
3125  ut_ad(!srv_read_only_mode);
3126 
3127  fts_msg_t* msg;
3128  os_event_t event;
3129 
3130  /* If there is an ongoing activity on dictionary, such as
3131  srv_master_evict_from_table_cache(), wait for it */
3132  dict_mutex_enter_for_mysql();
3133 
3134  /* Tells FTS optimizer system that we are exiting from
3135  optimizer thread, message send their after will not be
3136  processed */
3137  fts_opt_start_shutdown = true;
3138  dict_mutex_exit_for_mysql();
3139 
3140  /* We tell the OPTIMIZE thread to switch to state done, we
3141  can't delete the work queue here because the add thread needs
3142  deregister the FTS tables. */
3143  event = os_event_create();
3144 
3145  msg = fts_optimize_create_msg(FTS_MSG_STOP, NULL);
3146  msg->ptr = event;
3147 
3148  ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
3149 
3150  os_event_wait(event);
3151  os_event_free(event);
3152 
3153  ib_wqueue_free(fts_optimize_wq);
3154 
3155 }
3156 
3157 /**********************************************************************/
3159 UNIV_INTERN
3160 void
3161 fts_optimize_end(void)
3162 /*==================*/
3163 {
3164  ut_ad(!srv_read_only_mode);
3165 
3166  // FIXME: Potential race condition here: We should wait for
3167  // the optimize thread to confirm shutdown.
3168  fts_optimize_wq = NULL;
3169 }