uint fast_alter_partition_table(THD *thd, TABLE *table,
                                Alter_info *alter_info,
                                HA_CREATE_INFO *create_info,
                                TABLE_LIST *table_list,
                                char *db,
                                const char *table_name,
                                TABLE *fast_alter_table)
{
  /* Set-up struct used to write frm files */
  partition_info *part_info;
  ALTER_PARTITION_PARAM_TYPE lpt_obj;
  ALTER_PARTITION_PARAM_TYPE *lpt= &lpt_obj;
  bool action_completed= FALSE;
  bool close_table_on_failure= FALSE;
  bool frm_install= FALSE;
  MDL_ticket *mdl_ticket= table->mdl_ticket;
  DBUG_ASSERT(fast_alter_table);
  DBUG_ENTER("fast_alter_partition_table");

  part_info= fast_alter_table->part_info;
  lpt->thd= thd;
  lpt->table_list= table_list;
  lpt->part_info= part_info;
  lpt->alter_info= alter_info;
  lpt->create_info= create_info;
  lpt->db_options= create_info->table_options;
  if (create_info->row_type == ROW_TYPE_DYNAMIC)
    lpt->db_options|= HA_OPTION_PACK_RECORD;
  lpt->table= fast_alter_table;
  lpt->old_table= table;
  lpt->key_info_buffer= 0;
  lpt->key_count= 0;
  lpt->db= db;
  lpt->table_name= table_name;
  lpt->copied= 0;
  lpt->deleted= 0;
  lpt->pack_frm_data= NULL;
  lpt->pack_frm_len= 0;

  /* Never update timestamp columns when alter */
  lpt->table->timestamp_field_type= TIMESTAMP_NO_AUTO_SET;

  if (table->file->alter_table_flags(alter_info->flags) &
        HA_PARTITION_ONE_PHASE)
  {
    /*
      In the case where the engine supports one phase online partition
      changes it is not necessary to have any exclusive locks. The
      correctness is upheld instead by transactions being aborted if they
      access the table after its partition definition has changed (if they
      are still using the old partition definition).

      The handler is in this case responsible to ensure that all users
      start using the new frm file after it has changed. To implement
      one phase it is necessary for the handler to have the master copy
      of the frm file and use discovery mechanisms to renew it. Thus
      write frm will write the frm, pack the new frm and finally
      the frm is deleted and the discovery mechanisms will either restore
      back to the old or installing the new after the change is activated.

      Thus all open tables will be discovered that they are old, if not
      earlier as soon as they try an operation using the old table. One
      should ensure that this is checked already when opening a table,
      even if it is found in the cache of open tables.

      change_partitions will perform all operations and it is the duty of
      the handler to ensure that the frm files in the system gets updated
      in synch with the changes made and if an error occurs that a proper
      error handling is done.

      If the MySQL Server crashes at this moment but the handler succeeds
      in performing the change then the binlog is not written for the
      change. There is no way to solve this as long as the binlog is not
      transactional and even then it is hard to solve it completely.

      The first approach here was to downgrade locks. Now a different approach
      is decided upon. The idea is that the handler will have access to the
      Alter_info when store_lock arrives with TL_WRITE_ALLOW_READ. So if the
      handler knows that this functionality can be handled with a lower lock
      level it will set the lock level to TL_WRITE_ALLOW_WRITE immediately.
      Thus the need to downgrade the lock disappears.
      1) Write the new frm, pack it and then delete it
      2) Perform the change within the handler
    */
    if (mysql_write_frm(lpt, WFRM_WRITE_SHADOW | WFRM_PACK_FRM) ||
        mysql_change_partitions(lpt))
    {
      goto err;
    }
  }
  else if (alter_info->flags & ALTER_DROP_PARTITION)
  {
    /*
      Now after all checks and setting state on dropped partitions we can
      start the actual dropping of the partitions.

      Drop partition is actually two things happening. The first is that
      a lot of records are deleted. The second is that the behaviour of
      subsequent updates and writes and deletes will change. The delete
      part can be handled without any particular high lock level by
      transactional engines whereas non-transactional engines need to
      ensure that this change is done with an exclusive lock on the table.
      The second part, the change of partitioning does however require
      an exclusive lock to install the new partitioning as one atomic
      operation. If this is not the case, it is possible for two
      transactions to see the change in a different order than their
      serialisation order. Thus we need an exclusive lock for both
      transactional and non-transactional engines.

      For LIST partitions it could be possible to avoid the exclusive lock
      (and for RANGE partitions if they didn't rearrange range definitions
      after a DROP PARTITION) if one ensured that failed accesses to the
      dropped partitions was aborted for sure (thus only possible for
      transactional engines).

      0) Write an entry that removes the shadow frm file if crash occurs
      1) Write the new frm file as a shadow frm
      2) Get an exclusive metadata lock on the table (waits for all active
         transactions using this table). This ensures that we
         can release all other locks on the table and since no one can open
         the table, there can be no new threads accessing the table. They
         will be hanging on this exclusive lock.
      3) Write the ddl log to ensure that the operation is completed
         even in the presence of a MySQL Server crash (the log is executed
         before any other threads are started, so there are no locking issues).
      4) Close all tables that have already been opened but didn't stumble on
         the abort locked previously. This is done as part of the
         alter_close_tables call.
      5) Write the bin log
         Unfortunately the writing of the binlog is not synchronised with
         other logging activities. So no matter in which order the binlog
         is written compared to other activities there will always be cases
         where crashes make strange things occur. In this placement it can
         happen that the ALTER TABLE DROP PARTITION gets performed in the
         master but not in the slaves if we have a crash, after writing the
         ddl log but before writing the binlog. A solution to this would
         require writing the statement first in the ddl log and then
         when recovering from the crash read the binlog and insert it into
         the binlog if not written already.
      6) Install the previously written shadow frm file
      7) Prepare handlers for drop of partitions
      8) Drop the partitions
      9) Remove entries from ddl log
      10) Reopen table if under lock tables
      11) Complete query

      We insert Error injections at all places where it could be interesting
      to test if recovery is properly done.
    */
    if (write_log_drop_shadow_frm(lpt) ||
        ERROR_INJECT_CRASH("crash_drop_partition_1") ||
        ERROR_INJECT_ERROR("fail_drop_partition_1") ||
        mysql_write_frm(lpt, WFRM_WRITE_SHADOW) ||
        ERROR_INJECT_CRASH("crash_drop_partition_2") ||
        ERROR_INJECT_ERROR("fail_drop_partition_2") ||
        wait_while_table_is_used(thd, table, HA_EXTRA_FORCE_REOPEN) ||
        ERROR_INJECT_CRASH("crash_drop_partition_3") ||
        ERROR_INJECT_ERROR("fail_drop_partition_3") ||
        (close_table_on_failure= TRUE, FALSE) ||
        write_log_drop_partition(lpt) ||
        (action_completed= TRUE, FALSE) ||
        ERROR_INJECT_CRASH("crash_drop_partition_4") ||
        ERROR_INJECT_ERROR("fail_drop_partition_4") ||
        alter_close_tables(lpt, action_completed) ||
        (close_table_on_failure= FALSE, FALSE) ||
        ERROR_INJECT_CRASH("crash_drop_partition_5") ||
        ERROR_INJECT_ERROR("fail_drop_partition_5") ||
        ((!thd->lex->no_write_to_binlog) &&
         (write_bin_log(thd, FALSE,
                        thd->query(), thd->query_length()), FALSE)) ||
        ERROR_INJECT_CRASH("crash_drop_partition_6") ||
        ERROR_INJECT_ERROR("fail_drop_partition_6") ||
        (frm_install= TRUE, FALSE) ||
        mysql_write_frm(lpt, WFRM_INSTALL_SHADOW) ||
        (frm_install= FALSE, FALSE) ||
        ERROR_INJECT_CRASH("crash_drop_partition_7") ||
        ERROR_INJECT_ERROR("fail_drop_partition_7") ||
        mysql_drop_partitions(lpt) ||
        ERROR_INJECT_CRASH("crash_drop_partition_8") ||
        ERROR_INJECT_ERROR("fail_drop_partition_8") ||
        (write_log_completed(lpt, FALSE), FALSE) ||
        ERROR_INJECT_CRASH("crash_drop_partition_9") ||
        ERROR_INJECT_ERROR("fail_drop_partition_9") ||
        (alter_partition_lock_handling(lpt), FALSE))
    {
      handle_alter_part_error(lpt, action_completed, TRUE, frm_install,
                              close_table_on_failure);
      goto err;
    }
  }
  else if ((alter_info->flags & ALTER_ADD_PARTITION) &&
           (part_info->part_type == RANGE_PARTITION ||
            part_info->part_type == LIST_PARTITION))
  {
    /*
      ADD RANGE/LIST PARTITIONS
      In this case there are no tuples removed and no tuples are added.
      Thus the operation is merely adding a new partition. Thus it is
      necessary to perform the change as an atomic operation. Otherwise
      someone reading without seeing the new partition could potentially
      miss updates made by a transaction serialised before it that are
      inserted into the new partition.

      0) Write an entry that removes the shadow frm file if crash occurs
      1) Write the new frm file as a shadow frm file
      2) Get an exclusive metadata lock on the table (waits for all active
         transactions using this table). This ensures that we
         can release all other locks on the table and since no one can open
         the table, there can be no new threads accessing the table. They
         will be hanging on this exclusive lock.
      3) Write an entry to remove the new parttions if crash occurs
      4) Add the new partitions.
      5) Close all instances of the table and remove them from the table cache.
      6) Write binlog
      7) Now the change is completed except for the installation of the
         new frm file. We thus write an action in the log to change to
         the shadow frm file
      8) Install the new frm file of the table where the partitions are
         added to the table.
      9) Remove entries from ddl log
      10)Reopen tables if under lock tables
      11)Complete query
    */
    if (write_log_drop_shadow_frm(lpt) ||
        ERROR_INJECT_CRASH("crash_add_partition_1") ||
        ERROR_INJECT_ERROR("fail_add_partition_1") ||
        mysql_write_frm(lpt, WFRM_WRITE_SHADOW) ||
        ERROR_INJECT_CRASH("crash_add_partition_2") ||
        ERROR_INJECT_ERROR("fail_add_partition_2") ||
        wait_while_table_is_used(thd, table, HA_EXTRA_FORCE_REOPEN) ||
        ERROR_INJECT_CRASH("crash_add_partition_3") ||
        ERROR_INJECT_ERROR("fail_add_partition_3") ||
        (close_table_on_failure= TRUE, FALSE) ||
        write_log_add_change_partition(lpt) ||
        ERROR_INJECT_CRASH("crash_add_partition_4") ||
        ERROR_INJECT_ERROR("fail_add_partition_4") ||
        mysql_change_partitions(lpt) ||
        ERROR_INJECT_CRASH("crash_add_partition_5") ||
        ERROR_INJECT_ERROR("fail_add_partition_5") ||
        (close_table_on_failure= FALSE, FALSE) ||
        alter_close_tables(lpt, action_completed) ||
        ERROR_INJECT_CRASH("crash_add_partition_6") ||
        ERROR_INJECT_ERROR("fail_add_partition_6") ||
        ((!thd->lex->no_write_to_binlog) &&
         (write_bin_log(thd, FALSE,
                        thd->query(), thd->query_length()), FALSE)) ||
        ERROR_INJECT_CRASH("crash_add_partition_7") ||
        ERROR_INJECT_ERROR("fail_add_partition_7") ||
        write_log_rename_frm(lpt) ||
        (action_completed= TRUE, FALSE) ||
        ERROR_INJECT_CRASH("crash_add_partition_8") ||
        ERROR_INJECT_ERROR("fail_add_partition_8") ||
        (frm_install= TRUE, FALSE) ||
        mysql_write_frm(lpt, WFRM_INSTALL_SHADOW) ||
        (frm_install= FALSE, FALSE) ||
        ERROR_INJECT_CRASH("crash_add_partition_9") ||
        ERROR_INJECT_ERROR("fail_add_partition_9") ||
        (write_log_completed(lpt, FALSE), FALSE) ||
        ERROR_INJECT_CRASH("crash_add_partition_10") ||
        ERROR_INJECT_ERROR("fail_add_partition_10") ||
        (alter_partition_lock_handling(lpt), FALSE))
    {
      handle_alter_part_error(lpt, action_completed, FALSE, frm_install,
                              close_table_on_failure);
      goto err;
    }
  }
  else
  {
    /*
      ADD HASH PARTITION/
      COALESCE PARTITION/
      REBUILD PARTITION/
      REORGANIZE PARTITION

      In this case all records are still around after the change although
      possibly organised into new partitions, thus by ensuring that all
      updates go to both the old and the new partitioning scheme we can
      actually perform this operation lock-free. The only exception to
      this is when REORGANIZE PARTITION adds/drops ranges. In this case
      there needs to be an exclusive lock during the time when the range
      changes occur.
      This is only possible if the handler can ensure double-write for a
      period. The double write will ensure that it doesn't matter where the
      data is read from since both places are updated for writes. If such
      double writing is not performed then it is necessary to perform the
      change with the usual exclusive lock. With double writes it is even
      possible to perform writes in parallel with the reorganisation of
      partitions.

      Without double write procedure we get the following procedure.
      The only difference with using double write is that we can downgrade
      the lock to TL_WRITE_ALLOW_WRITE. Double write in this case only
      double writes from old to new. If we had double writing in both
      directions we could perform the change completely without exclusive
      lock for HASH partitions.
      Handlers that perform double writing during the copy phase can actually
      use a lower lock level. This can be handled inside store_lock in the
      respective handler.

      0) Write an entry that removes the shadow frm file if crash occurs
      1) Write the shadow frm file of new partitioning
      2) Log such that temporary partitions added in change phase are
         removed in a crash situation
      3) Add the new partitions
         Copy from the reorganised partitions to the new partitions
      4) Get an exclusive metadata lock on the table (waits for all active
         transactions using this table). This ensures that we
         can release all other locks on the table and since no one can open
         the table, there can be no new threads accessing the table. They
         will be hanging on this exclusive lock.
      5) Log that operation is completed and log all complete actions
         needed to complete operation from here
      6) Write bin log
      7) Close all instances of the table and remove them from the table cache.
      8) Prepare handlers for rename and delete of partitions
      9) Rename and drop the reorged partitions such that they are no
         longer used and rename those added to their real new names.
      10) Install the shadow frm file
      11) Reopen the table if under lock tables
      12) Complete query
    */
    if (write_log_drop_shadow_frm(lpt) ||
        ERROR_INJECT_CRASH("crash_change_partition_1") ||
        ERROR_INJECT_ERROR("fail_change_partition_1") ||
        mysql_write_frm(lpt, WFRM_WRITE_SHADOW) ||
        ERROR_INJECT_CRASH("crash_change_partition_2") ||
        ERROR_INJECT_ERROR("fail_change_partition_2") ||
        (close_table_on_failure= TRUE, FALSE) ||
        write_log_add_change_partition(lpt) ||
        ERROR_INJECT_CRASH("crash_change_partition_3") ||
        ERROR_INJECT_ERROR("fail_change_partition_3") ||
        mysql_change_partitions(lpt) ||
        ERROR_INJECT_CRASH("crash_change_partition_4") ||
        ERROR_INJECT_ERROR("fail_change_partition_4") ||
        wait_while_table_is_used(thd, table, HA_EXTRA_FORCE_REOPEN) ||
        ERROR_INJECT_CRASH("crash_change_partition_5") ||
        ERROR_INJECT_ERROR("fail_change_partition_5") ||
        write_log_final_change_partition(lpt) ||
        (action_completed= TRUE, FALSE) ||
        ERROR_INJECT_CRASH("crash_change_partition_6") ||
        ERROR_INJECT_ERROR("fail_change_partition_6") ||
        ((!thd->lex->no_write_to_binlog) &&
         (write_bin_log(thd, FALSE,
                        thd->query(), thd->query_length()), FALSE)) ||
        ERROR_INJECT_CRASH("crash_change_partition_7") ||
        ERROR_INJECT_ERROR("fail_change_partition_7") ||
        ((frm_install= TRUE), FALSE) ||
        mysql_write_frm(lpt, WFRM_INSTALL_SHADOW) ||
        (frm_install= FALSE, FALSE) ||
        ERROR_INJECT_CRASH("crash_change_partition_8") ||
        ERROR_INJECT_ERROR("fail_change_partition_8") ||
        alter_close_tables(lpt, action_completed) ||
        (close_table_on_failure= FALSE, FALSE) ||
        ERROR_INJECT_CRASH("crash_change_partition_9") ||
        ERROR_INJECT_ERROR("fail_change_partition_9") ||
        mysql_drop_partitions(lpt) ||
        ERROR_INJECT_CRASH("crash_change_partition_10") ||
        ERROR_INJECT_ERROR("fail_change_partition_10") ||
        mysql_rename_partitions(lpt) ||
        ERROR_INJECT_CRASH("crash_change_partition_11") ||
        ERROR_INJECT_ERROR("fail_change_partition_11") ||
        (write_log_completed(lpt, FALSE), FALSE) ||
        ERROR_INJECT_CRASH("crash_change_partition_12") ||
        ERROR_INJECT_ERROR("fail_change_partition_12") ||
        (alter_partition_lock_handling(lpt), FALSE))
    {
      handle_alter_part_error(lpt, action_completed, FALSE, frm_install,
                              close_table_on_failure);
      goto err;
    }
  }
  downgrade_mdl_if_lock_tables_mode(thd, mdl_ticket, MDL_SHARED_NO_READ_WRITE);
  /*
    A final step is to write the query to the binlog and send ok to the
    user
  */
  DBUG_RETURN(fast_end_partition(thd, lpt->copied, lpt->deleted, table_list));
err:
  if (action_completed)
  {
    /*
      Although error occurred, the action was forced to retry for completion.
      Therefore we must close+reopen all instances of the table.
    */
    (void) alter_partition_lock_handling(lpt);
  }
  else
  {
    /*
      The failed action was reverted, leave the original table as is and
      close/destroy the intermediate table object and its share.
    */
    close_temporary(lpt->table, 1, 0);
  }
  downgrade_mdl_if_lock_tables_mode(thd, mdl_ticket, MDL_SHARED_NO_READ_WRITE);
  DBUG_RETURN(TRUE);
}


uint ha_partition::max_supported_record_length() const
{
  return min_of_the_max_uint(&handler::max_supported_record_length);
}


uint32 get_next_partition_id_list(PARTITION_ITERATOR *part_iter)
{
  if (part_iter->part_nums.cur >= part_iter->part_nums.end)
  {
    if (part_iter->ret_null_part)
    {
      part_iter->ret_null_part= FALSE;
      return part_iter->part_info->has_null_part_id;
    }
    part_iter->part_nums.cur= part_iter->part_nums.start;
    part_iter->ret_null_part= part_iter->ret_null_part_orig;
    return NOT_A_PARTITION_ID;
  }
  else
  {
    partition_info *part_info= part_iter->part_info;
    uint32 num_part= part_iter->part_nums.cur++;
    if (part_info->column_list)
    {
      uint num_columns= part_info->part_field_list.elements;
      return part_info->list_col_array[num_part*num_columns].partition_id;
    }
    return part_info->list_array[num_part].partition_id;
  }
}


int ha_partition::delete_table(const char *name)
{
  DBUG_ENTER("ha_partition::delete_table");

  DBUG_RETURN(del_ren_cre_table(name, NULL, NULL, NULL));
}


int ha_partition::truncate()
{
  int error;
  handler **file;
  DBUG_ENTER("ha_partition::truncate");

  /*
    TRUNCATE also means resetting auto_increment. Hence, reset
    it so that it will be initialized again at the next use.
  */
  lock_auto_increment();
  table_share->ha_part_data->next_auto_inc_val= 0;
  table_share->ha_part_data->auto_inc_initialized= FALSE;
  unlock_auto_increment();

  file= m_file;
  do
  {
    if ((error= (*file)->ha_truncate()))
      DBUG_RETURN(error);
  } while (*(++file));
  DBUG_RETURN(0);
}


int get_partition_id_key_sub(partition_info *part_info,
                             uint32 *part_id)
{
  longlong func_value;
  *part_id= get_part_id_key(part_info->subpart_field_array,
                            part_info->num_subparts, &func_value);
  return FALSE;
}


void get_full_part_id_from_key(const TABLE *table, uchar *buf,
                               KEY *key_info,
                               const key_range *key_spec,
                               part_id_range *part_spec)
{
  bool result;
  partition_info *part_info= table->part_info;
  uchar *rec0= table->record[0];
  longlong func_value;
  DBUG_ENTER("get_full_part_id_from_key");

  key_restore(buf, (uchar*)key_spec->key, key_info, key_spec->length);
  if (likely(rec0 == buf))
  {
    result= part_info->get_partition_id(part_info, &part_spec->start_part,
                                        &func_value);
  }
  else
  {
    Field **part_field_array= part_info->full_part_field_array;
    set_field_ptr(part_field_array, buf, rec0);
    result= part_info->get_partition_id(part_info, &part_spec->start_part,
                                        &func_value);
    set_field_ptr(part_field_array, rec0, buf);
  }
  part_spec->end_part= part_spec->start_part;
  if (unlikely(result))
    part_spec->start_part++;
  DBUG_VOID_RETURN;
}


int ha_partition::rename_table(const char *from, const char *to)
{
  DBUG_ENTER("ha_partition::rename_table");

  DBUG_RETURN(del_ren_cre_table(from, to, NULL, NULL));
}


int ha_partition::cmp_ref(const uchar *ref1, const uchar *ref2)
{
  int cmp;
  my_ptrdiff_t diff1, diff2;
  DBUG_ENTER("ha_partition::cmp_ref");

  cmp = m_file[0]->cmp_ref((ref1 + PARTITION_BYTES_IN_POS),
			   (ref2 + PARTITION_BYTES_IN_POS));
  if (cmp)
    DBUG_RETURN(cmp);

  if ((ref1[0] == ref2[0]) && (ref1[1] == ref2[1]))
  {
   /* This means that the references are same and are in same partition.*/
    DBUG_RETURN(0);
  }

  /*
    In Innodb we compare with either primary key value or global DB_ROW_ID so
    it is not possible that the two references are equal and are in different
    partitions, but in myisam it is possible since we are comparing offsets.
    Remove this assert if DB_ROW_ID is changed to be per partition.
  */
  DBUG_ASSERT(!m_innodb);

  diff1= ref2[1] - ref1[1];
  diff2= ref2[0] - ref1[0];
  if (diff1 > 0)
  {
    DBUG_RETURN(-1);
  }
  if (diff1 < 0)
  {
    DBUG_RETURN(+1);
  }
  if (diff2 > 0)
  {
    DBUG_RETURN(-1);
  }
  DBUG_RETURN(+1);
}


static uint32 calculate_key_value(Field **field_array)
{
  ulong nr1= 1;
  ulong nr2= 4;
  bool use_51_hash;
  use_51_hash= test((*field_array)->table->part_info->key_algorithm ==
                    partition_info::KEY_ALGORITHM_51);

  do
  {
    Field *field= *field_array;
    if (use_51_hash)
    {
      switch (field->real_type()) {
      case MYSQL_TYPE_TINY:
      case MYSQL_TYPE_SHORT:
      case MYSQL_TYPE_LONG:
      case MYSQL_TYPE_FLOAT:
      case MYSQL_TYPE_DOUBLE:
      case MYSQL_TYPE_NEWDECIMAL:
      case MYSQL_TYPE_TIMESTAMP:
      case MYSQL_TYPE_LONGLONG:
      case MYSQL_TYPE_INT24:
      case MYSQL_TYPE_TIME:
      case MYSQL_TYPE_DATETIME:
      case MYSQL_TYPE_YEAR:
      case MYSQL_TYPE_NEWDATE:
        {
          if (field->is_null())
          {
            nr1^= (nr1 << 1) | 1;
            continue;
          }
          /* Force this to my_hash_sort_bin, which was used in 5.1! */
          uint len= field->pack_length();
          my_charset_bin.coll->hash_sort(&my_charset_bin, field->ptr, len,
                                         &nr1, &nr2);
          /* Done with this field, continue with next one. */
          continue;
        }
      case MYSQL_TYPE_STRING:
      case MYSQL_TYPE_VARCHAR:
      case MYSQL_TYPE_BIT:
        /* Not affected, same in 5.1 and 5.5 */
        break;
      /*
        ENUM/SET uses my_hash_sort_simple in 5.1 (i.e. my_charset_latin1)
        and my_hash_sort_bin in 5.5!
      */
      case MYSQL_TYPE_ENUM:
      case MYSQL_TYPE_SET:
        {
          if (field->is_null())
          {
            nr1^= (nr1 << 1) | 1;
            continue;
          }
          /* Force this to my_hash_sort_bin, which was used in 5.1! */
          uint len= field->pack_length();
          my_charset_latin1.coll->hash_sort(&my_charset_latin1, field->ptr,
                                            len, &nr1, &nr2);
          continue;
        }
      /* These types should not be allowed for partitioning! */
      case MYSQL_TYPE_NULL:
      case MYSQL_TYPE_DECIMAL:
      case MYSQL_TYPE_DATE:
      case MYSQL_TYPE_TINY_BLOB:
      case MYSQL_TYPE_MEDIUM_BLOB:
      case MYSQL_TYPE_LONG_BLOB:
      case MYSQL_TYPE_BLOB:
      case MYSQL_TYPE_VAR_STRING:
      case MYSQL_TYPE_GEOMETRY:
        /* fall through. */
      default:
        DBUG_ASSERT(0);                    // New type?
        /* Fall through for default hashing (5.5). */
      }
      /* fall through, use collation based hashing. */
    }
    field->hash(&nr1, &nr2);
  } while (*(++field_array));
  return (uint32) nr1;
}


static uint alter_table_flags(uint flags __attribute__((unused)))
{
  return (HA_PARTITION_FUNCTION_SUPPORTED |
          HA_FAST_CHANGE_PARTITION);
}


static bool write_log_rename_frm(ALTER_PARTITION_PARAM_TYPE *lpt)
{
  partition_info *part_info= lpt->part_info;
  DDL_LOG_MEMORY_ENTRY *log_entry;
  DDL_LOG_MEMORY_ENTRY *exec_log_entry= part_info->exec_log_entry;
  char path[FN_REFLEN + 1];
  char shadow_path[FN_REFLEN + 1];
  DDL_LOG_MEMORY_ENTRY *old_first_log_entry= part_info->first_log_entry;
  DBUG_ENTER("write_log_rename_frm");

  part_info->first_log_entry= NULL;
  build_table_filename(path, sizeof(path) - 1, lpt->db,
                       lpt->table_name, "", 0);
  build_table_shadow_filename(shadow_path, sizeof(shadow_path) - 1, lpt);
  mysql_mutex_lock(&LOCK_gdl);
  if (write_log_replace_delete_frm(lpt, 0UL, shadow_path, path, TRUE))
    goto error;
  log_entry= part_info->first_log_entry;
  part_info->frm_log_entry= log_entry;
  if (write_execute_ddl_log_entry(log_entry->entry_pos,
                                    FALSE, &exec_log_entry))
    goto error;
  release_part_info_log_entries(old_first_log_entry);
  mysql_mutex_unlock(&LOCK_gdl);
  DBUG_RETURN(FALSE);

error:
  release_part_info_log_entries(part_info->first_log_entry);
  mysql_mutex_unlock(&LOCK_gdl);
  part_info->first_log_entry= old_first_log_entry;
  part_info->frm_log_entry= NULL;
  my_error(ER_DDL_LOG_ERROR, MYF(0));
  DBUG_RETURN(TRUE);
}


int get_part_iter_for_interval_cols_via_map(partition_info *part_info,
                                            bool is_subpart,
                                            uint32 *store_length_array,
                                            uchar *min_value, uchar *max_value,
                                            uint min_len, uint max_len,
                                            uint flags,
                                            PARTITION_ITERATOR *part_iter)
{
  uint32 nparts;
  get_col_endpoint_func  get_col_endpoint;
  DBUG_ENTER("get_part_iter_for_interval_cols_via_map");

  if (part_info->part_type == RANGE_PARTITION)
  {
    get_col_endpoint= get_partition_id_cols_range_for_endpoint;
    part_iter->get_next= get_next_partition_id_range;
  }
  else if (part_info->part_type == LIST_PARTITION)
  {
    get_col_endpoint= get_partition_id_cols_list_for_endpoint;
    part_iter->get_next= get_next_partition_id_list;
    part_iter->part_info= part_info;
    DBUG_ASSERT(part_info->num_list_values);
  }
  else
    assert(0);

  if (flags & NO_MIN_RANGE)
    part_iter->part_nums.start= part_iter->part_nums.cur= 0;
  else
  {
    // Copy from min_value to record
    nparts= store_tuple_to_record(part_info->part_field_array,
                                  store_length_array,
                                  min_value,
                                  min_value + min_len);
    part_iter->part_nums.start= part_iter->part_nums.cur=
      get_col_endpoint(part_info, TRUE, !(flags & NEAR_MIN),
                       nparts);
  }
  if (flags & NO_MAX_RANGE)
  {
    if (part_info->part_type == RANGE_PARTITION)
      part_iter->part_nums.end= part_info->num_parts;
    else /* LIST_PARTITION */
    {
      DBUG_ASSERT(part_info->part_type == LIST_PARTITION);
      part_iter->part_nums.end= part_info->num_list_values;
    }
  }
  else
  {
    // Copy from max_value to record
    nparts= store_tuple_to_record(part_info->part_field_array,
                                  store_length_array,
                                  max_value,
                                  max_value + max_len);
    part_iter->part_nums.end= get_col_endpoint(part_info, FALSE,
                                               !(flags & NEAR_MAX),
                                               nparts);
  }
  if (part_iter->part_nums.start == part_iter->part_nums.end)
    DBUG_RETURN(0);
  DBUG_RETURN(1);
}


void ha_partition::late_extra_no_cache(uint partition_id)
{
  handler *file;
  DBUG_ENTER("ha_partition::late_extra_no_cache");

  if (!m_extra_cache && !m_extra_prepare_for_update)
    DBUG_VOID_RETURN;
  file= m_file[partition_id];
  (void) file->extra(HA_EXTRA_NO_CACHE);
  DBUG_ASSERT(partition_id == m_extra_cache_part_id);
  m_extra_cache_part_id= NO_CURRENT_PART_ID;
  DBUG_VOID_RETURN;
}


bool ha_partition::create_handler_file(const char *name)
{
  partition_element *part_elem, *subpart_elem;
  uint i, j, part_name_len, subpart_name_len;
  uint tot_partition_words, tot_name_len, num_parts;
  uint tot_parts= 0;
  uint tot_len_words, tot_len_byte, chksum, tot_name_words;
  char *name_buffer_ptr;
  uchar *file_buffer, *engine_array;
  bool result= TRUE;
  char file_name[FN_REFLEN];
  char part_name[FN_REFLEN];
  char subpart_name[FN_REFLEN];
  File file;
  List_iterator_fast <partition_element> part_it(m_part_info->partitions);
  DBUG_ENTER("create_handler_file");

  num_parts= m_part_info->partitions.elements;
  DBUG_PRINT("info", ("table name = %s, num_parts = %u", name,
                      num_parts));
  tot_name_len= 0;
  for (i= 0; i < num_parts; i++)
  {
    part_elem= part_it++;
    if (part_elem->part_state != PART_NORMAL &&
        part_elem->part_state != PART_TO_BE_ADDED &&
        part_elem->part_state != PART_CHANGED)
      continue;
    tablename_to_filename(part_elem->partition_name, part_name,
                          FN_REFLEN);
    part_name_len= strlen(part_name);
    if (!m_is_sub_partitioned)
    {
      tot_name_len+= part_name_len + 1;
      tot_parts++;
    }
    else
    {
      List_iterator_fast <partition_element> sub_it(part_elem->subpartitions);
      for (j= 0; j < m_part_info->num_subparts; j++)
      {
	subpart_elem= sub_it++;
        tablename_to_filename(subpart_elem->partition_name,
                              subpart_name,
                              FN_REFLEN);
	subpart_name_len= strlen(subpart_name);
	tot_name_len+= part_name_len + subpart_name_len + 5;
        tot_parts++;
      }
    }
  }
  /*
     File format:
     Length in words              4 byte
     Checksum                     4 byte
     Total number of partitions   4 byte
     Array of engine types        n * 4 bytes where
     n = (m_tot_parts + 3)/4
     Length of name part in bytes 4 bytes
     (Names in filename format)
     Name part                    m * 4 bytes where
     m = ((length_name_part + 3)/4)*4

     All padding bytes are zeroed
  */
  tot_partition_words= (tot_parts + PAR_WORD_SIZE - 1) / PAR_WORD_SIZE;
  tot_name_words= (tot_name_len + PAR_WORD_SIZE - 1) / PAR_WORD_SIZE;
  /* 4 static words (tot words, checksum, tot partitions, name length) */
  tot_len_words= 4 + tot_partition_words + tot_name_words;
  tot_len_byte= PAR_WORD_SIZE * tot_len_words;
  if (!(file_buffer= (uchar *) my_malloc(tot_len_byte, MYF(MY_ZEROFILL))))
    DBUG_RETURN(TRUE);
  engine_array= (file_buffer + PAR_ENGINES_OFFSET);
  name_buffer_ptr= (char*) (engine_array + tot_partition_words * PAR_WORD_SIZE
                            + PAR_WORD_SIZE);
  part_it.rewind();
  for (i= 0; i < num_parts; i++)
  {
    part_elem= part_it++;
    if (part_elem->part_state != PART_NORMAL &&
        part_elem->part_state != PART_TO_BE_ADDED &&
        part_elem->part_state != PART_CHANGED)
      continue;
    if (!m_is_sub_partitioned)
    {
      tablename_to_filename(part_elem->partition_name, part_name, FN_REFLEN);
      name_buffer_ptr= strmov(name_buffer_ptr, part_name)+1;
      *engine_array= (uchar) ha_legacy_type(part_elem->engine_type);
      DBUG_PRINT("info", ("engine: %u", *engine_array));
      engine_array++;
    }
    else
    {
      List_iterator_fast <partition_element> sub_it(part_elem->subpartitions);
      for (j= 0; j < m_part_info->num_subparts; j++)
      {
	subpart_elem= sub_it++;
        tablename_to_filename(part_elem->partition_name, part_name,
                              FN_REFLEN);
        tablename_to_filename(subpart_elem->partition_name, subpart_name,
                              FN_REFLEN);
	name_buffer_ptr+= name_add(name_buffer_ptr,
				   part_name,
				   subpart_name);
        *engine_array= (uchar) ha_legacy_type(subpart_elem->engine_type);
        DBUG_PRINT("info", ("engine: %u", *engine_array));
	engine_array++;
      }
    }
  }
  chksum= 0;
  int4store(file_buffer, tot_len_words);
  int4store(file_buffer + PAR_NUM_PARTS_OFFSET, tot_parts);
  int4store(file_buffer + PAR_ENGINES_OFFSET +
            (tot_partition_words * PAR_WORD_SIZE),
            tot_name_len);
  for (i= 0; i < tot_len_words; i++)
    chksum^= uint4korr(file_buffer + PAR_WORD_SIZE * i);
  int4store(file_buffer + PAR_CHECKSUM_OFFSET, chksum);
  /*
    Add .par extension to the file name.
    Create and write and close file
    to be used at open, delete_table and rename_table
  */
  fn_format(file_name, name, "", ha_par_ext, MY_APPEND_EXT);
  if ((file= mysql_file_create(key_file_partition,
                               file_name, CREATE_MODE, O_RDWR | O_TRUNC,
                               MYF(MY_WME))) >= 0)
  {
    result= mysql_file_write(file, (uchar *) file_buffer, tot_len_byte,
                             MYF(MY_WME | MY_NABP)) != 0;
    (void) mysql_file_close(file, MYF(0));
  }
  else
    result= TRUE;
  my_free(file_buffer);
  DBUG_RETURN(result);
}


static bool write_log_drop_partition(ALTER_PARTITION_PARAM_TYPE *lpt)
{
  partition_info *part_info= lpt->part_info;
  DDL_LOG_MEMORY_ENTRY *log_entry;
  DDL_LOG_MEMORY_ENTRY *exec_log_entry= part_info->exec_log_entry;
  char tmp_path[FN_REFLEN + 1];
  char path[FN_REFLEN + 1];
  uint next_entry= 0;
  DDL_LOG_MEMORY_ENTRY *old_first_log_entry= part_info->first_log_entry;
  DBUG_ENTER("write_log_drop_partition");

  part_info->first_log_entry= NULL;
  build_table_filename(path, sizeof(path) - 1, lpt->db,
                       lpt->table_name, "", 0);
  build_table_shadow_filename(tmp_path, sizeof(tmp_path) - 1, lpt);
  mysql_mutex_lock(&LOCK_gdl);
  if (write_log_dropped_partitions(lpt, &next_entry, (const char*)path,
                                   FALSE))
    goto error;
  if (write_log_replace_delete_frm(lpt, next_entry, (const char*)tmp_path,
                                  (const char*)path, TRUE))
    goto error;
  log_entry= part_info->first_log_entry;
  part_info->frm_log_entry= log_entry;
  if (write_execute_ddl_log_entry(log_entry->entry_pos,
                                    FALSE, &exec_log_entry))
    goto error;
  release_part_info_log_entries(old_first_log_entry);
  mysql_mutex_unlock(&LOCK_gdl);
  DBUG_RETURN(FALSE);

error:
  release_part_info_log_entries(part_info->first_log_entry);
  mysql_mutex_unlock(&LOCK_gdl);
  part_info->first_log_entry= old_first_log_entry;
  part_info->frm_log_entry= NULL;
  my_error(ER_DDL_LOG_ERROR, MYF(0));
  DBUG_RETURN(TRUE);
}


int ha_partition::extra(enum ha_extra_function operation)
{
  DBUG_ENTER("ha_partition:extra");
  DBUG_PRINT("info", ("operation: %d", (int) operation));

  switch (operation) {
    /* Category 1), used by most handlers */
  case HA_EXTRA_KEYREAD:
  case HA_EXTRA_NO_KEYREAD:
  case HA_EXTRA_FLUSH:
    DBUG_RETURN(loop_extra(operation));

    /* Category 2), used by non-MyISAM handlers */
  case HA_EXTRA_IGNORE_DUP_KEY:
  case HA_EXTRA_NO_IGNORE_DUP_KEY:
  case HA_EXTRA_KEYREAD_PRESERVE_FIELDS:
  {
    if (!m_myisam)
      DBUG_RETURN(loop_extra(operation));
    break;
  }

  /* Category 3), used by MyISAM handlers */
  case HA_EXTRA_PREPARE_FOR_RENAME:
    DBUG_RETURN(prepare_for_rename());
    break;
  case HA_EXTRA_PREPARE_FOR_UPDATE:
    /*
      Needs to be run on the first partition in the range now, and
      later in late_extra_cache, when switching to a new partition to scan.
    */
    m_extra_prepare_for_update= TRUE;
    if (m_part_spec.start_part != NO_CURRENT_PART_ID)
    {
      if (!m_extra_cache)
        m_extra_cache_part_id= m_part_spec.start_part;
      DBUG_ASSERT(m_extra_cache_part_id == m_part_spec.start_part);
      (void) m_file[m_part_spec.start_part]->extra(HA_EXTRA_PREPARE_FOR_UPDATE);
    }
    break;
  case HA_EXTRA_NORMAL:
  case HA_EXTRA_QUICK:
  case HA_EXTRA_FORCE_REOPEN:
  case HA_EXTRA_PREPARE_FOR_DROP:
  case HA_EXTRA_FLUSH_CACHE:
  {
    if (m_myisam)
      DBUG_RETURN(loop_extra(operation));
    break;
  }
  case HA_EXTRA_NO_READCHECK:
  {
    /*
      This is only done as a part of ha_open, which is also used in
      ha_partition::open, so no need to do anything.
    */
    break;
  }
  case HA_EXTRA_CACHE:
  {
    prepare_extra_cache(0);
    break;
  }
  case HA_EXTRA_NO_CACHE:
  {
    int ret= 0;
    if (m_extra_cache_part_id != NO_CURRENT_PART_ID)
      ret= m_file[m_extra_cache_part_id]->extra(HA_EXTRA_NO_CACHE);
    m_extra_cache= FALSE;
    m_extra_cache_size= 0;
    m_extra_prepare_for_update= FALSE;
    m_extra_cache_part_id= NO_CURRENT_PART_ID;
    DBUG_RETURN(ret);
  }
  case HA_EXTRA_WRITE_CACHE:
  {
    m_extra_cache= FALSE;
    m_extra_cache_size= 0;
    m_extra_prepare_for_update= FALSE;
    m_extra_cache_part_id= NO_CURRENT_PART_ID;
    DBUG_RETURN(loop_extra(operation));
  }
  case HA_EXTRA_IGNORE_NO_KEY:
  case HA_EXTRA_NO_IGNORE_NO_KEY:
  {
    /*
      Ignore as these are specific to NDB for handling
      idempotency
     */
    break;
  }
  case HA_EXTRA_WRITE_CAN_REPLACE:
  case HA_EXTRA_WRITE_CANNOT_REPLACE:
  {
    /*
      Informs handler that write_row() can replace rows which conflict
      with row being inserted by PK/unique key without reporting error
      to the SQL-layer.

      This optimization is not safe for partitioned table in general case
      since we may have to put new version of row into partition which is
      different from partition in which old version resides (for example
      when we partition by non-PK column or by some column which is not
      part of unique key which were violated).
      And since NDB which is the only engine at the moment that supports
      this optimization handles partitioning on its own we simple disable
      it here. (BTW for NDB this optimization is safe since it supports
      only KEY partitioning and won't use this optimization for tables
      which have additional unique constraints).
    */
    break;
  }
    /* Category 7), used by federated handlers */
  case HA_EXTRA_INSERT_WITH_UPDATE:
    DBUG_RETURN(loop_extra(operation));
    /* Category 8) Operations only used by NDB */
  case HA_EXTRA_DELETE_CANNOT_BATCH:
  case HA_EXTRA_UPDATE_CANNOT_BATCH:
  {
    /* Currently only NDB use the *_CANNOT_BATCH */
    break;
  }
    /* Category 9) Operations only used by MERGE */
  case HA_EXTRA_ADD_CHILDREN_LIST:
  case HA_EXTRA_ATTACH_CHILDREN:
  case HA_EXTRA_IS_ATTACHED_CHILDREN:
  case HA_EXTRA_DETACH_CHILDREN:
  {
    /* Special actions for MERGE tables. Ignore. */
    break;
  }
  /*
    http://dev.mysql.com/doc/refman/5.1/en/partitioning-limitations.html
    says we no longer support logging to partitioned tables, so we fail
    here.
  */
  case HA_EXTRA_MARK_AS_LOG_TABLE:
    DBUG_RETURN(ER_UNSUPORTED_LOG_ENGINE);
  default:
  {
    /* Temporary crash to discover what is wrong */
    DBUG_ASSERT(0);
    break;
  }
  }
  DBUG_RETURN(0);
}


int ha_partition::rnd_end()
{
  handler **file;
  DBUG_ENTER("ha_partition::rnd_end");
  switch (m_scan_value) {
  case 2:                                       // Error
    break;
  case 1:
    if (NO_CURRENT_PART_ID != m_part_spec.start_part)         // Table scan
    {
      late_extra_no_cache(m_part_spec.start_part);
      m_file[m_part_spec.start_part]->ha_rnd_end();
    }
    break;
  case 0:
    file= m_file;
    do
    {
      if (bitmap_is_set(&(m_part_info->used_partitions), (file - m_file)))
        (*file)->ha_rnd_end();
    } while (*(++file));
    break;
  }
  m_scan_value= 2;
  m_part_spec.start_part= NO_CURRENT_PART_ID;
  DBUG_RETURN(0);
}


static CURLcode imap_state_listsearch_resp(struct connectdata *conn,
                                           int imapcode,
                                           imapstate instate)
{
  CURLcode result = CURLE_OK;
  char *line = conn->data->state.buffer;
  size_t len = strlen(line);

  (void)instate; /* No use for this yet */

  if(imapcode == '*') {
    /* Temporarily add the LF character back and send as body to the client */
    line[len] = '\n';
    result = Curl_client_write(conn, CLIENTWRITE_BODY, line, len + 1);
    line[len] = '\0';
  }
  else if(imapcode != IMAP_RESP_OK)
    result = CURLE_QUOTE_ERROR; /* TODO: Fix error code */
  else
    /* End of DO phase */
    state(conn, IMAP_STOP);

  return result;
}


static int imap_getsock(struct connectdata *conn, curl_socket_t *socks,
                        int numsocks)
{
  return Curl_pp_getsock(&conn->proto.imapc.pp, socks, numsocks);
}


static CURLcode imap_perform_authentication(struct connectdata *conn)
{
  CURLcode result = CURLE_OK;
  struct imap_conn *imapc = &conn->proto.imapc;
  saslprogress progress;

  /* Check if already authenticated OR if there is enough data to authenticate
     with and end the connect phase if we don't */
  if(imapc->preauth ||
     !Curl_sasl_can_authenticate(&imapc->sasl, conn)) {
    state(conn, IMAP_STOP);
    return result;
  }

  /* Calculate the SASL login details */
  result = Curl_sasl_start(&imapc->sasl, conn, imapc->ir_supported, &progress);

  if(!result) {
    if(progress == SASL_INPROGRESS)
      state(conn, IMAP_AUTHENTICATE);
    else if(!imapc->login_disabled && (imapc->preftype & IMAP_TYPE_CLEARTEXT))
      /* Perform clear text authentication */
      result = imap_perform_login(conn);
    else {
      /* Other mechanisms not supported */
      infof(conn->data, "No known authentication mechanisms supported!\n");
      result = CURLE_LOGIN_DENIED;
    }
  }

  return result;
}


static CURLcode imap_perform_authenticate(struct connectdata *conn,
                                          const char *mech,
                                          const char *initresp)
{
  CURLcode result = CURLE_OK;

  if(initresp) {
    /* Send the AUTHENTICATE command with the initial response */
    result = imap_sendf(conn, "AUTHENTICATE %s %s", mech, initresp);
  }
  else {
    /* Send the AUTHENTICATE command */
    result = imap_sendf(conn, "AUTHENTICATE %s", mech);
  }

  return result;
}


is_dir (char const *filename, int flags, glob_t const *pglob)
{
  struct stat st;
  struct_stat64 st64;
  return (__glibc_unlikely (flags & GLOB_ALTDIRFUNC)
          ? pglob->gl_stat (filename, &st) == 0 && S_ISDIR (st.st_mode)
          : __stat64 (filename, &st64) == 0 && S_ISDIR (st64.st_mode));
}


check_file_output (const struct url *u, struct http_stat *hs,
                   struct response *resp, char *hdrval, size_t hdrsize)
{
  /* Determine the local filename if needed. Notice that if -O is used
   * hstat.local_file is set by http_loop to the argument of -O. */
  if (!hs->local_file)
    {
      char *local_file = NULL;

      /* Honor Content-Disposition whether possible. */
      if (!opt.content_disposition
          || !resp_header_copy (resp, "Content-Disposition",
                                hdrval, hdrsize)
          || !parse_content_disposition (hdrval, &local_file))
        {
          /* The Content-Disposition header is missing or broken.
           * Choose unique file name according to given URL. */
          hs->local_file = url_file_name (u, NULL);
        }
      else
        {
          DEBUGP (("Parsed filename from Content-Disposition: %s\n",
                  local_file));
          hs->local_file = url_file_name (u, local_file);
        }

      xfree (local_file);
    }

  hs->temporary = opt.delete_after || opt.spider || !acceptable (hs->local_file);
  if (hs->temporary)
    {
      char *tmp = aprintf ("%s.tmp", hs->local_file);
      xfree (hs->local_file);
      hs->local_file = tmp;
    }

  /* TODO: perform this check only once. */
  if (!hs->existence_checked && file_exists_p (hs->local_file, NULL))
    {
      if (opt.noclobber && !opt.output_document)
        {
          /* If opt.noclobber is turned on and file already exists, do not
             retrieve the file. But if the output_document was given, then this
             test was already done and the file didn't exist. Hence the !opt.output_document */
          return RETRUNNEEDED;
        }
      else if (!ALLOW_CLOBBER)
        {
          char *unique = unique_name (hs->local_file, true);
          if (unique != hs->local_file)
            xfree (hs->local_file);
          hs->local_file = unique;
        }
    }
  hs->existence_checked = true;

  /* Support timestamping */
  if (opt.timestamping && !hs->timestamp_checked)
    {
      uerr_t timestamp_err = set_file_timestamp (hs);
      if (timestamp_err != RETROK)
        return timestamp_err;
    }
  return RETROK;
}


create_authorization_line (const char *au, const char *user,
                           const char *passwd, const char *method,
                           const char *path, bool *finished, uerr_t *auth_err)
{
  /* We are called only with known schemes, so we can dispatch on the
     first letter. */
  switch (c_toupper (*au))
    {
    case 'B':                   /* Basic */
      *finished = true;
      return basic_authentication_encode (user, passwd);
#ifdef ENABLE_DIGEST
    case 'D':                   /* Digest */
      *finished = true;
      return digest_authentication_encode (au, user, passwd, method, path, auth_err);
#endif
#ifdef ENABLE_NTLM
    case 'N':                   /* NTLM */
      if (!ntlm_input (&pconn.ntlm, au))
        {
          *finished = true;
          return NULL;
        }
      return ntlm_output (&pconn.ntlm, user, passwd, finished);
#endif
    default:
      /* We shouldn't get here -- this function should be only called
         with values approved by known_authentication_scheme_p.  */
      abort ();
    }
}


fd_read_body (const char *downloaded_filename, int fd, FILE *out, wgint toread, wgint startpos,

              wgint *qtyread, wgint *qtywritten, double *elapsed, int flags,
              FILE *out2)
{
  int ret = 0;
#undef max
#define max(a,b) ((a) > (b) ? (a) : (b))
  int dlbufsize = max (BUFSIZ, 8 * 1024);
  char *dlbuf = xmalloc (dlbufsize);

  struct ptimer *timer = NULL;
  double last_successful_read_tm = 0;

  /* The progress gauge, set according to the user preferences. */
  void *progress = NULL;

  /* Non-zero if the progress gauge is interactive, i.e. if it can
     continually update the display.  When true, smaller timeout
     values are used so that the gauge can update the display when
     data arrives slowly. */
  bool progress_interactive = false;

  bool exact = !!(flags & rb_read_exactly);

  /* Used only by HTTP/HTTPS chunked transfer encoding.  */
  bool chunked = flags & rb_chunked_transfer_encoding;
  wgint skip = 0;

  /* How much data we've read/written.  */
  wgint sum_read = 0;
  wgint sum_written = 0;
  wgint remaining_chunk_size = 0;

#ifdef HAVE_LIBZ
  /* try to minimize the number of calls to inflate() and write_data() per
     call to fd_read() */
  unsigned int gzbufsize = dlbufsize * 4;
  char *gzbuf = NULL;
  z_stream gzstream;

  if (flags & rb_compressed_gzip)
    {
      gzbuf = xmalloc (gzbufsize);
      if (gzbuf != NULL)
        {
          gzstream.zalloc = zalloc;
          gzstream.zfree = zfree;
          gzstream.opaque = Z_NULL;
          gzstream.next_in = Z_NULL;
          gzstream.avail_in = 0;

          #define GZIP_DETECT 32 /* gzip format detection */
          #define GZIP_WINDOW 15 /* logarithmic window size (default: 15) */
          ret = inflateInit2 (&gzstream, GZIP_DETECT | GZIP_WINDOW);
          if (ret != Z_OK)
            {
              xfree (gzbuf);
              errno = (ret == Z_MEM_ERROR) ? ENOMEM : EINVAL;
              ret = -1;
              goto out;
            }
        }
      else
        {
          errno = ENOMEM;
          ret = -1;
          goto out;
        }
    }
#endif

  if (flags & rb_skip_startpos)
    skip = startpos;

  if (opt.show_progress)
    {
      const char *filename_progress;
      /* If we're skipping STARTPOS bytes, pass 0 as the INITIAL
         argument to progress_create because the indicator doesn't
         (yet) know about "skipping" data.  */
      wgint start = skip ? 0 : startpos;
      if (opt.dir_prefix)
        filename_progress = downloaded_filename + strlen (opt.dir_prefix) + 1;
      else
        filename_progress = downloaded_filename;
      progress = progress_create (filename_progress, start, start + toread);
      progress_interactive = progress_interactive_p (progress);
    }

  if (opt.limit_rate)
    limit_bandwidth_reset ();

  /* A timer is needed for tracking progress, for throttling, and for
     tracking elapsed time.  If either of these are requested, start
     the timer.  */
  if (progress || opt.limit_rate || elapsed)
    {
      timer = ptimer_new ();
      last_successful_read_tm = 0;
    }

  /* Use a smaller buffer for low requested bandwidths.  For example,
     with --limit-rate=2k, it doesn't make sense to slurp in 16K of
     data and then sleep for 8s.  With buffer size equal to the limit,
     we never have to sleep for more than one second.  */
  if (opt.limit_rate && opt.limit_rate < dlbufsize)
    dlbufsize = opt.limit_rate;

  /* Read from FD while there is data to read.  Normally toread==0
     means that it is unknown how much data is to arrive.  However, if
     EXACT is set, then toread==0 means what it says: that no data
     should be read.  */
  while (!exact || (sum_read < toread))
    {
      int rdsize;
      double tmout = opt.read_timeout;

      if (chunked)
        {
          if (remaining_chunk_size == 0)
            {
              char *line = fd_read_line (fd);
              char *endl;
              if (line == NULL)
                {
                  ret = -1;
                  break;
                }
              else if (out2 != NULL)
                fwrite (line, 1, strlen (line), out2);

              remaining_chunk_size = strtol (line, &endl, 16);
              xfree (line);

              if (remaining_chunk_size < 0)
                {
                  ret = -1;
                  break;
                }

              if (remaining_chunk_size == 0)
                {
                  ret = 0;
                  line = fd_read_line (fd);
                  if (line == NULL)
                    ret = -1;
                  else
                    {
                      if (out2 != NULL)
                        fwrite (line, 1, strlen (line), out2);
                      xfree (line);
                    }
                  break;
                }
            }

          rdsize = MIN (remaining_chunk_size, dlbufsize);
        }
      else
        rdsize = exact ? MIN (toread - sum_read, dlbufsize) : dlbufsize;

      if (progress_interactive)
        {
          /* For interactive progress gauges, always specify a ~1s
             timeout, so that the gauge can be updated regularly even
             when the data arrives very slowly or stalls.  */
          tmout = 0.95;
          if (opt.read_timeout)
            {
              double waittm;
              waittm = ptimer_read (timer) - last_successful_read_tm;
              if (waittm + tmout > opt.read_timeout)
                {
                  /* Don't let total idle time exceed read timeout. */
                  tmout = opt.read_timeout - waittm;
                  if (tmout < 0)
                    {
                      /* We've already exceeded the timeout. */
                      ret = -1, errno = ETIMEDOUT;
                      break;
                    }
                }
            }
        }
      ret = fd_read (fd, dlbuf, rdsize, tmout);

      if (progress_interactive && ret < 0 && errno == ETIMEDOUT)
        ret = 0;                /* interactive timeout, handled above */
      else if (ret <= 0)
        break;                  /* EOF or read error */

      if (progress || opt.limit_rate || elapsed)
        {
          ptimer_measure (timer);
          if (ret > 0)
            last_successful_read_tm = ptimer_read (timer);
        }

      if (ret > 0)
        {
          int write_res;

          sum_read += ret;

#ifdef HAVE_LIBZ
          if (gzbuf != NULL)
            {
              int err;
              int towrite;
              gzstream.avail_in = ret;
              gzstream.next_in = (unsigned char *) dlbuf;

              do
                {
                  gzstream.avail_out = gzbufsize;
                  gzstream.next_out = (unsigned char *) gzbuf;

                  err = inflate (&gzstream, Z_NO_FLUSH);

                  switch (err)
                    {
                    case Z_MEM_ERROR:
                      errno = ENOMEM;
                      ret = -1;
                      goto out;
                    case Z_NEED_DICT:
                    case Z_DATA_ERROR:
                      errno = EINVAL;
                      ret = -1;
                      goto out;
                    case Z_STREAM_END:
                      if (exact && sum_read != toread)
                        {
                          DEBUGP(("zlib stream ended unexpectedly after "
                                  "%ld/%ld bytes\n", sum_read, toread));
                        }
                    }

                  towrite = gzbufsize - gzstream.avail_out;
                  write_res = write_data (out, out2, gzbuf, towrite, &skip,
                                          &sum_written);
                  if (write_res < 0)
                    {
                      ret = (write_res == -3) ? -3 : -2;
                      goto out;
                    }
                }
              while (gzstream.avail_out == 0);
            }
          else
#endif
            {
              write_res = write_data (out, out2, dlbuf, ret, &skip,
                                      &sum_written);
              if (write_res < 0)
                {
                  ret = (write_res == -3) ? -3 : -2;
                  goto out;
                }
            }

          if (chunked)
            {
              remaining_chunk_size -= ret;
              if (remaining_chunk_size == 0)
                {
                  char *line = fd_read_line (fd);
                  if (line == NULL)
                    {
                      ret = -1;
                      break;
                    }
                  else
                    {
                      if (out2 != NULL)
                        fwrite (line, 1, strlen (line), out2);
                      xfree (line);
                    }
                }
            }
        }

      if (opt.limit_rate)
        limit_bandwidth (ret, timer);

      if (progress)
        progress_update (progress, ret, ptimer_read (timer));
#ifdef WINDOWS
      if (toread > 0 && opt.show_progress)
        ws_percenttitle (100.0 *
                         (startpos + sum_read) / (startpos + toread));
#endif
    }
  if (ret < -1)
    ret = -1;

 out:
  if (progress)
    progress_finish (progress, ptimer_read (timer));

  if (elapsed)
    *elapsed = ptimer_read (timer);
  if (timer)
    ptimer_destroy (timer);

#ifdef HAVE_LIBZ
  if (gzbuf != NULL)
    {
      int err = inflateEnd (&gzstream);
      if (ret >= 0)
        {
          /* with compression enabled, ret must be 0 if successful */
          if (err == Z_OK)
            ret = 0;
          else
            {
              errno = EINVAL;
              ret = -1;
            }
        }
      xfree (gzbuf);

      if (gzstream.total_in != sum_read)
        {
          DEBUGP(("zlib read size differs from raw read size (%lu/%lu)\n",
                  gzstream.total_in, sum_read));
        }
    }
#endif

  if (qtyread)
    *qtyread += sum_read;
  if (qtywritten)
    *qtywritten += sum_written;

  xfree (dlbuf);

  return ret;
}


x509_proxy_expiration_time( const char *proxy_file )
{
#if !defined(HAVE_EXT_GLOBUS)
	(void) proxy_file;
	set_error_string( NOT_SUPPORTED_MSG );
	return -1;
#else

	time_t expiration_time = -1;
	globus_gsi_cred_handle_t proxy_handle = x509_proxy_read( proxy_file );

	if ( proxy_handle == NULL ) {
		return -1;
	}

	expiration_time = x509_proxy_expiration_time( proxy_handle );

	x509_proxy_free( proxy_handle );

	return expiration_time;

#endif /* !defined(GSS_AUTHENTICATION) */
}


void parse_resource_manager_string( const char *string, char **host,
									char **port, char **service,
									char **subject )
{
	char *p;
	char *q;
	size_t len = strlen( string );

	char *my_host = (char *)calloc( len+1, sizeof(char) );
	char *my_port = (char *)calloc( len+1, sizeof(char) );
	char *my_service = (char *)calloc( len+1, sizeof(char) );
	char *my_subject = (char *)calloc( len+1, sizeof(char) );
	ASSERT( my_host && my_port && my_service && my_subject );

	p = my_host;
	q = my_host;

	while ( *string != '\0' ) {
		if ( *string == ':' ) {
			if ( q == my_host ) {
				p = my_port;
				q = my_port;
				string++;
			} else if ( q == my_port || q == my_service ) {
				p = my_subject;
				q = my_subject;
				string++;
			} else {
				*(p++) = *(string++);
			}
		} else if ( *string == '/' ) {
			if ( q == my_host || q == my_port ) {
				p = my_service;
				q = my_service;
				string++;
			} else {
				*(p++) = *(string++);
			}
		} else {
			*(p++) = *(string++);
		}
	}

	if ( host != NULL ) {
		*host = my_host;
	} else {
		free( my_host );
	}

	if ( port != NULL ) {
		*port = my_port;
	} else {
		free( my_port );
	}

	if ( service != NULL ) {
		*service = my_service;
	} else {
		free( my_service );
	}

	if ( subject != NULL ) {
		*subject = my_subject;
	} else {
		free( my_subject );
	}
}


instruct(PInfo pi, const char *target, Attr attrs, const char *content) {
    if (0 == strcmp("xml", target)) {
#if HAS_ENCODING_SUPPORT
	for (; 0 != attrs->name; attrs++) {
	    if (0 == strcmp("encoding", attrs->name)) {
		pi->options->rb_enc = rb_enc_find(attrs->value);
	    }
	}
#elif HAS_PRIVATE_ENCODING
	for (; 0 != attrs->name; attrs++) {
	    if (0 == strcmp("encoding", attrs->name)) {
		pi->options->rb_enc = rb_str_new2(attrs->value);
	    }
	}
#endif
    }
}


aspath_str_update (struct aspath *as)
{
  if (as->str)
    XFREE (MTYPE_AS_STR, as->str);
  aspath_make_str_count (as);
}


String *Item_func_conv_charset::val_str(String *str)
{
  DBUG_ASSERT(fixed == 1);
  if (use_cached_value)
    return null_value ? 0 : &str_value;
  String *arg= args[0]->val_str(str);
  uint dummy_errors;
  if (!arg)
  {
    null_value=1;
    return 0;
  }
  null_value= tmp_value.copy(arg->ptr(), arg->length(), arg->charset(),
                             conv_charset, &dummy_errors);
  return null_value ? 0 : check_well_formed_result(&tmp_value,
                                                   false, // send warning
                                                   true); // truncate
}


String *Item_func_ltrim::val_str(String *str)
{
  DBUG_ASSERT(fixed == 1);
  char buff[MAX_FIELD_WIDTH], *ptr, *end;
  String tmp(buff,sizeof(buff),system_charset_info);
  String *res, *remove_str;
  uint remove_length;
  LINT_INIT(remove_length);

  res= args[0]->val_str(str);
  if ((null_value=args[0]->null_value))
    return 0;
  remove_str= &remove;                          /* Default value. */
  if (arg_count == 2)
  {
    remove_str= args[1]->val_str(&tmp);
    if ((null_value= args[1]->null_value))
      return 0;
  }

  if ((remove_length= remove_str->length()) == 0 ||
      remove_length > res->length())
    return res;

  ptr= (char*) res->ptr();
  end= ptr+res->length();
#ifdef USE_MB
  if (use_mb(res->charset()))
  {
    ptr= trim_left_mb(res, ptr, end, remove_str);
  }
  else
#endif /* USE_MB */
  {
    if (remove_length == 1)
    {
      char chr=(*remove_str)[0];
      while (ptr != end && *ptr == chr)
        ptr++;
    }
    else
    {
      const char *r_ptr=remove_str->ptr();
      end-=remove_length;
      while (ptr <= end && !memcmp(ptr, r_ptr, remove_length))
        ptr+=remove_length;
      end+=remove_length;
    }
  }
  if (ptr == res->ptr())
    return res;
  tmp_value.set(*res,(uint) (ptr - res->ptr()),(uint) (end-ptr));
  return &tmp_value;
}


String *Item_func_aes_decrypt::val_str(String *str)
{
  DBUG_ASSERT(fixed == 1);
  char key_buff[80];
  String tmp_key_value(key_buff, sizeof(key_buff), system_charset_info);
  String *sptr, *key;
  THD *thd= current_thd;
  ulong aes_opmode;
  iv_argument iv_arg;
  DBUG_ENTER("Item_func_aes_decrypt::val_str");

  sptr= args[0]->val_str(str);			// String to decrypt
  key=  args[1]->val_str(&tmp_key_value);	// Key
  aes_opmode= thd->variables.my_aes_mode;
  DBUG_ASSERT(aes_opmode <= MY_AES_END);

  if (sptr && key)  			// Need to have both arguments not NULL
  {
    const unsigned char *iv_str=
      iv_arg.retrieve_iv_ptr((enum my_aes_opmode) aes_opmode, arg_count, args,
      func_name(), thd, &null_value);
    if (null_value)
      DBUG_RETURN(NULL);
    str_value.set_charset(&my_charset_bin);
    if (!str_value.alloc(sptr->length()))  // Ensure that memory is free
    {
      // finally decrypt directly to allocated buffer.
      int length;
      length= my_aes_decrypt((unsigned char *) sptr->ptr(), sptr->length(),
                             (unsigned char *) str_value.ptr(),
                             (unsigned char *) key->ptr(), key->length(),
                             (enum my_aes_opmode) aes_opmode, iv_str);
      if (length >= 0)  // if we got correct data data
      {
        str_value.length((uint) length);
        DBUG_RETURN(&str_value);
      }
    }
  }
  // Bad parameters. No memory or bad data will all go here
  null_value=1;
  DBUG_RETURN(0);
}


void Item_func_sha2::fix_length_and_dec()
{
  maybe_null = 1;
  max_length = 0;

#if defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY)
  int sha_variant= args[1]->const_item() ? args[1]->val_int() : 512;

  switch (sha_variant) {
#ifndef OPENSSL_NO_SHA512
  case 512:
    fix_length_and_charset(SHA512_DIGEST_LENGTH * 2, default_charset());
    break;
  case 384:
    fix_length_and_charset(SHA384_DIGEST_LENGTH * 2, default_charset());
    break;
#endif
#ifndef OPENSSL_NO_SHA256
  case 256:
  case 0: // SHA-256 is the default
    fix_length_and_charset(SHA256_DIGEST_LENGTH * 2, default_charset());
    break;
  case 224:
    fix_length_and_charset(SHA224_DIGEST_LENGTH * 2, default_charset());
    break;
#endif
  default:
    push_warning_printf(current_thd,
      Sql_condition::WARN_LEVEL_WARN,
      ER_WRONG_PARAMETERS_TO_NATIVE_FCT,
      ER(ER_WRONG_PARAMETERS_TO_NATIVE_FCT), "sha2");
  }

  CHARSET_INFO *cs= get_checksum_charset(args[0]->collation.collation->csname);
  args[0]->collation.set(cs, DERIVATION_COERCIBLE);

#else
  push_warning_printf(current_thd,
    Sql_condition::WARN_LEVEL_WARN,
    ER_FEATURE_DISABLED,
    ER(ER_FEATURE_DISABLED),
    "sha2", "--with-ssl");
#endif /* defined(HAVE_OPENSSL) && !defined(EMBEDDED_LIBRARY) */
}


String *Item_func_insert::val_str(String *str)
{
  DBUG_ASSERT(fixed == 1);
  String *res,*res2;
  longlong start, length, orig_len;  /* must be longlong to avoid truncation */

  null_value=0;
  res=args[0]->val_str(str);
  res2=args[3]->val_str(&tmp_value);
  start= args[1]->val_int();
  length= args[2]->val_int();

  if (args[0]->null_value || args[1]->null_value || args[2]->null_value ||
      args[3]->null_value)
    goto null; /* purecov: inspected */

  orig_len= static_cast<longlong>(res->length());

  if ((start < 1) || (start > orig_len))
    return res;                                 // Wrong param; skip insert

  --start;    // Internal start from '0'

  if ((length < 0) || (length > orig_len))
    length= orig_len;

  /*
    There is one exception not handled (intentionaly) by the character set
    aggregation code. If one string is strong side and is binary, and
    another one is weak side and is a multi-byte character string,
    then we need to operate on the second string in terms on bytes when
    calling ::numchars() and ::charpos(), rather than in terms of characters.
    Lets substitute its character set to binary.
  */
  if (collation.collation == &my_charset_bin)
  {
    res->set_charset(&my_charset_bin);
    res2->set_charset(&my_charset_bin);
  }

  /* start and length are now sufficiently valid to pass to charpos function */
   start= res->charpos((int) start);
   length= res->charpos((int) length, (uint32) start);

  /* Re-testing with corrected params */
  if (start > orig_len)
    return res; /* purecov: inspected */        // Wrong param; skip insert
  if (length > orig_len - start)
    length= orig_len - start;

  if ((ulonglong) (orig_len - length + res2->length()) >
      (ulonglong) current_thd->variables.max_allowed_packet)
  {
    push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN,
			ER_WARN_ALLOWED_PACKET_OVERFLOWED,
			ER(ER_WARN_ALLOWED_PACKET_OVERFLOWED),
			func_name(), current_thd->variables.max_allowed_packet);
    goto null;
  }
  if (res->uses_buffer_owned_by(str))
  {
    if (tmp_value_res.alloc(orig_len) ||
        tmp_value_res.copy(*res))
      goto null;
    res= &tmp_value_res;
  }
  else
    res= copy_if_not_alloced(str, res, orig_len);

  res->replace((uint32) start,(uint32) length,*res2);
  return res;
null:
  null_value=1;
  return 0;
}


void Item_func_make_set::update_used_tables()
{
  Item_func::update_used_tables();
  item->update_used_tables();
  used_tables_cache|=item->used_tables();
  const_item_cache&=item->const_item();
  with_subselect= item->has_subquery();
  with_stored_program= item->has_stored_program();
}


String *Item_func_rpad::val_str(String *str)
{
  DBUG_ASSERT(fixed == 1);
  uint32 res_byte_length,res_char_length,pad_char_length,pad_byte_length;
  char *to;
  const char *ptr_pad;
  /* must be longlong to avoid truncation */
  longlong count= args[1]->val_int();
  longlong byte_count;
  String *res= args[0]->val_str(str);
  String *rpad= args[2]->val_str(&rpad_str);

  if (!res || args[1]->null_value || !rpad ||
      ((count < 0) && !args[1]->unsigned_flag))
    goto err;
  null_value=0;
  /* Assumes that the maximum length of a String is < INT_MAX32. */
  /* Set here so that rest of code sees out-of-bound value as such. */
  if ((ulonglong) count > INT_MAX32)
    count= INT_MAX32;
  /*
    There is one exception not handled (intentionaly) by the character set
    aggregation code. If one string is strong side and is binary, and
    another one is weak side and is a multi-byte character string,
    then we need to operate on the second string in terms on bytes when
    calling ::numchars() and ::charpos(), rather than in terms of characters.
    Lets substitute its character set to binary.
  */
  if (collation.collation == &my_charset_bin)
  {
    res->set_charset(&my_charset_bin);
    rpad->set_charset(&my_charset_bin);
  }

#ifdef USE_MB
  if (use_mb(rpad->charset()))
  {
    // This will chop off any trailing illegal characters from rpad.
    String *well_formed_pad= args[2]->check_well_formed_result(rpad,
                                                               false, //send warning
                                                               true); //truncate
    if (!well_formed_pad)
      goto err;
  }
#endif

  if (count <= (res_char_length= res->numchars()))
  {						// String to pad is big enough
    res->length(res->charpos((int) count));	// Shorten result if longer
    return (res);
  }
  pad_char_length= rpad->numchars();

  byte_count= count * collation.collation->mbmaxlen;
  if ((ulonglong) byte_count > current_thd->variables.max_allowed_packet)
  {
    push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN,
			ER_WARN_ALLOWED_PACKET_OVERFLOWED,
			ER(ER_WARN_ALLOWED_PACKET_OVERFLOWED),
			func_name(), current_thd->variables.max_allowed_packet);
    goto err;
  }
  if (args[2]->null_value || !pad_char_length)
    goto err;
  res_byte_length= res->length();	/* Must be done before alloc_buffer */
  if (!(res= alloc_buffer(res,str,&tmp_value, (ulong) byte_count)))
    goto err;

  to= (char*) res->ptr()+res_byte_length;
  ptr_pad=rpad->ptr();
  pad_byte_length= rpad->length();
  count-= res_char_length;
  for ( ; (uint32) count > pad_char_length; count-= pad_char_length)
  {
    memcpy(to,ptr_pad,pad_byte_length);
    to+= pad_byte_length;
  }
  if (count)
  {
    pad_byte_length= rpad->charpos((int) count);
    memcpy(to,ptr_pad,(size_t) pad_byte_length);
    to+= pad_byte_length;
  }
  res->length((uint) (to- (char*) res->ptr()));
  return (res);

 err:
  null_value=1;
  return 0;
}


String *Item_func_concat_ws::val_str(String *str)
{
  DBUG_ASSERT(fixed == 1);
  char tmp_str_buff[10];
  String tmp_sep_str(tmp_str_buff, sizeof(tmp_str_buff),default_charset_info),
         *sep_str, *res, *res2,*use_as_buff;
  uint i;
  bool is_const= 0;

  null_value=0;
  if (!(sep_str= args[0]->val_str(&tmp_sep_str)))
    goto null;

  use_as_buff= &tmp_value;
  str->length(0);				// QQ; Should be removed
  res=str;

  // Skip until non-null argument is found.
  // If not, return the empty string
  for (i=1; i < arg_count; i++)
    if ((res= args[i]->val_str(str)))
    {
      is_const= args[i]->const_item() || !args[i]->used_tables();
      break;
    }

  if (i ==  arg_count)
    return make_empty_result();

  for (i++; i < arg_count ; i++)
  {
    if (!(res2= args[i]->val_str(use_as_buff)))
      continue;					// Skip NULL

    if (res->length() + sep_str->length() + res2->length() >
	current_thd->variables.max_allowed_packet)
    {
      push_warning_printf(current_thd, Sql_condition::WARN_LEVEL_WARN,
			  ER_WARN_ALLOWED_PACKET_OVERFLOWED,
			  ER(ER_WARN_ALLOWED_PACKET_OVERFLOWED), func_name(),
			  current_thd->variables.max_allowed_packet);
      goto null;
    }
    if (!is_const && res->alloced_length() >=
	res->length() + sep_str->length() + res2->length())
    {						// Use old buffer
      res->append(*sep_str);			// res->length() > 0 always
      res->append(*res2);
    }
    else if (str->alloced_length() >=
	     res->length() + sep_str->length() + res2->length())
    {
      /* We have room in str;  We can't get any errors here */
      if (str->ptr() == res2->ptr())
      {						// This is quite uncommon!
	str->replace(0,0,*sep_str);
	str->replace(0,0,*res);
      }
      else
      {
        // If res2 is a substring of str, then clone it first.
        char buff[STRING_BUFFER_USUAL_SIZE];
        String res2_clone(buff, sizeof(buff), system_charset_info);
        if (res2->uses_buffer_owned_by(str))
        {
          if (res2_clone.copy(*res2))
            goto null;
          res2= &res2_clone;
        }
        str->copy(*res);
        str->append(*sep_str);
        str->append(*res2);
      }
      res=str;
      use_as_buff= &tmp_value;
    }
    else if (res == &tmp_value)
    {
      if (res->append(*sep_str) || res->append(*res2))
	goto null; // Must be a blob
    }
    else if (res2 == &tmp_value)
    {						// This can happend only 1 time
      if (tmp_value.replace(0,0,*sep_str) || tmp_value.replace(0,0,*res))
	goto null;
      res= &tmp_value;
      use_as_buff=str;				// Put next arg here
    }
    else if (tmp_value.is_alloced() && res2->ptr() >= tmp_value.ptr() &&
	     res2->ptr() < tmp_value.ptr() + tmp_value.alloced_length())
    {
      /*
	This happens really seldom:
	In this case res2 is sub string of tmp_value.  We will
	now work in place in tmp_value to set it to res | sep_str | res2
      */
      /* Chop the last characters in tmp_value that isn't in res2 */
      tmp_value.length((uint32) (res2->ptr() - tmp_value.ptr()) +
		       res2->length());
      /* Place res2 at start of tmp_value, remove chars before res2 */
      if (tmp_value.replace(0,(uint32) (res2->ptr() - tmp_value.ptr()),
			    *res) ||
	  tmp_value.replace(res->length(),0, *sep_str))
	goto null;
      res= &tmp_value;
      use_as_buff=str;			// Put next arg here
    }
    else
    {						// Two big const strings
      /*
        NOTE: We should be prudent in the initial allocation unit -- the
        size of the arguments is a function of data distribution, which can
        be any. Instead of overcommitting at the first row, we grow the
        allocated amount by the factor of 2. This ensures that no more than
        25% of memory will be overcommitted on average.
      */

      uint concat_len= res->length() + sep_str->length() + res2->length();

      if (tmp_value.alloced_length() < concat_len)
      {
        if (tmp_value.alloced_length() == 0)
        {
          if (tmp_value.alloc(concat_len))
            goto null;
        }
        else
        {
          uint new_len = max(tmp_value.alloced_length() * 2, concat_len);

          if (tmp_value.realloc(new_len))
            goto null;
        }
      }

      if (tmp_value.copy(*res) ||
	  tmp_value.append(*sep_str) ||
	  tmp_value.append(*res2))
	goto null;
      res= &tmp_value;
      use_as_buff=str;
    }
  }
  res->set_charset(collation.collation);
  return res;

null:
  null_value=1;
  return 0;
}


static void mark_tree_contents_uninteresting(struct tree *tree)
{
	struct tree_desc desc;
	struct name_entry entry;
	struct object *obj = &tree->object;

	if (!has_object_file(&obj->oid))
		return;
	if (parse_tree(tree) < 0)
		die("bad tree %s", oid_to_hex(&obj->oid));

	init_tree_desc(&desc, tree->buffer, tree->size);
	while (tree_entry(&desc, &entry)) {
		switch (object_type(entry.mode)) {
		case OBJ_TREE:
			mark_tree_uninteresting(lookup_tree(entry.oid->hash));
			break;
		case OBJ_BLOB:
			mark_blob_uninteresting(lookup_blob(entry.oid->hash));
			break;
		default:
			/* Subproject commit - not in this repository */
			break;
		}
	}

	/*
	 * We don't care about the tree any more
	 * after it has been marked uninteresting.
	 */
	free_tree_buffer(tree);
}


static void theme_unref(THEME_REC *rec)
{
	if (--rec->refcount == 0)
		theme_real_destroy(rec);
}


static void event_hosthidden(IRC_SERVER_REC *server, const char *data)
{
	char *params, *newhost, *p, *newuserhost;

	g_return_if_fail(server != NULL);

	params = event_get_params(data, 2, NULL, &newhost);
	/* do a sanity check */
	if (!strchr(newhost, '*') && !strchr(newhost, '?') &&
			!strchr(newhost, '!') && !strchr(newhost, '#') &&
			!strchr(newhost, '&') && !strchr(newhost, ' ') &&
			*newhost != '\0' && *newhost != '@' &&
			*newhost != ':' && *newhost != '-' &&
			newhost[strlen(newhost) - 1] != '-') {
		if (strchr(newhost, '@')) {
			newuserhost = g_strdup(newhost);
			g_free(server->userhost);
			server->userhost = newuserhost;
		} else if (server->userhost != NULL) {
			/* no user@, only process if we know the user@
			 * already
			 */
			p = strchr(server->userhost, '@');
			if (p == NULL)
				p = server->userhost;
			newuserhost = g_strdup_printf("%.*s@%s", (int)(p - server->userhost), server->userhost, newhost);
			g_free(server->userhost);
			server->userhost = newuserhost;
		}
	}
	g_free(params);
}


static void sig_complete_format(GList **list, WINDOW_REC *window,
				const char *word, const char *line, int *want_space)
{
	const char *ptr;
	int words;

	g_return_if_fail(list != NULL);
	g_return_if_fail(word != NULL);
	g_return_if_fail(line != NULL);

        ptr = line;

	words = 0;
	if (*ptr != '\0') {
		do {
			ptr++;
			words++;
			ptr = strchr(ptr, ' ');
		} while (ptr != NULL);
	}

	if (words > 2)
		return;

	*list = completion_get_formats(line, word);
	if (*list != NULL) signal_stop();
}


static void dcc_send_connect(SEND_DCC_REC *dcc)
{
	dcc->handle = dcc_connect_ip(&dcc->addr, dcc->port);

	if (dcc->handle != NULL) {
		dcc->starttime = time(NULL);

		dcc->tagread = g_input_add(dcc->handle, G_INPUT_READ,
					   (GInputFunction) dcc_send_read_size,
					   dcc);
		dcc->tagwrite = g_input_add(dcc->handle, G_INPUT_WRITE,
					    (GInputFunction) dcc_send_data,
					    dcc);
		signal_emit("dcc connected", 1, dcc);
	} else {
		/* error connecting */
		signal_emit("dcc error connect", 1, dcc);
		dcc_destroy(DCC(dcc));
	}
}


static void dcc_chat_connect(CHAT_DCC_REC *dcc)
{
	g_return_if_fail(IS_DCC_CHAT(dcc));

	if (dcc->addrstr[0] == '\0' ||
	    dcc->starttime != 0 || dcc->handle != NULL) {
		/* already sent a chat request / already chatting */
		return;
	}

	dcc->handle = dcc_connect_ip(&dcc->addr, dcc->port);
	if (dcc->handle != NULL) {
		dcc->tagconn = g_input_add(dcc->handle,
					   G_INPUT_WRITE | G_INPUT_READ,
					   (GInputFunction) sig_chat_connected, dcc);
	} else {
		/* error connecting */
		signal_emit("dcc error connect", 1, dcc);
		dcc_destroy(DCC(dcc));
	}
}


THEME_REC *theme_create(const char *path, const char *name)
{
	THEME_REC *rec;

	g_return_val_if_fail(path != NULL, NULL);
	g_return_val_if_fail(name != NULL, NULL);

	rec = g_new0(THEME_REC, 1);
	rec->refcount = 1;
	rec->path = g_strdup(path);
	rec->name = g_strdup(name);
	rec->abstracts = g_hash_table_new((GHashFunc) g_str_hash,
					  (GCompareFunc) g_str_equal);
	rec->modules = g_hash_table_new((GHashFunc) g_istr_hash,
					(GCompareFunc) g_istr_equal);
	themes = g_slist_append(themes, rec);
	signal_emit("theme created", 1, rec);

	return rec;
}


static void dcc_send_connected(SEND_DCC_REC *dcc)
{
        GIOChannel *handle;
	IPADDR addr;
	int port;

	/* accept connection */
	handle = net_accept(dcc->handle, &addr, &port);
	if (handle == NULL)
		return;

	/* TODO: some kind of paranoia check would be nice. it would check
	   that the host of the nick who we sent the request matches the
	   address who connected us. */

	net_disconnect(dcc->handle);
	g_source_remove(dcc->tagconn);
        dcc->tagconn = -1;

	dcc->starttime = time(NULL);
	dcc->handle = handle;
	memcpy(&dcc->addr, &addr, sizeof(IPADDR));
	net_ip2host(&dcc->addr, dcc->addrstr);
	dcc->port = port;

	dcc->tagread = g_input_add(handle, G_INPUT_READ,
				   (GInputFunction) dcc_send_read_size, dcc);
	dcc->tagwrite = g_input_add(handle, G_INPUT_WRITE,
				    (GInputFunction) dcc_send_data, dcc);

	signal_emit("dcc connected", 1, dcc);
}


static void send_message(SERVER_REC *server, const char *target,
			 const char *msg, int target_type)
{
	IRC_SERVER_REC *ircserver;
	CHANNEL_REC *channel;
	char *str;
	char *recoded;

        ircserver = IRC_SERVER(server);
	g_return_if_fail(ircserver != NULL);
	g_return_if_fail(target != NULL);
	g_return_if_fail(msg != NULL);

	if (*target == '!') {
		/* !chan -> !12345chan */
		channel = channel_find(server, target);
		if (channel != NULL &&
		    g_ascii_strcasecmp(channel->name, target) != 0)
			target = channel->name;
	}

	recoded = recode_out(SERVER(server), msg, target);
	str = g_strdup_printf("PRIVMSG %s :%s", target, recoded);
	irc_send_cmd_split(ircserver, str, 2, ircserver->max_msgs_in_cmd);
	g_free(str);
	g_free(recoded);
}


void dcc_send_deinit(void)
{
	dcc_queue_deinit();

        dcc_unregister_type("SEND");
	signal_remove("dcc destroyed", (SIGNAL_FUNC) sig_dcc_destroyed);
	signal_remove("dcc reply send pasv", (SIGNAL_FUNC) dcc_send_connect);
	command_unbind("dcc send", (SIGNAL_FUNC) cmd_dcc_send);
}


static void theme_save(THEME_REC *theme, int save_all)
{
	CONFIG_REC *config;
	THEME_SAVE_REC data;
	char *path;
	char *basename;
	int ok;

	config = config_open(theme->path, -1);
        if (config != NULL)
                config_parse(config);
        else {
                if (g_ascii_strcasecmp(theme->name, "default") == 0) {
                        config = config_open(NULL, -1);
                        config_parse_data(config, default_theme, "internal");
                        config_change_file_name(config, theme->path, 0660);
                } else {
                        config = config_open(theme->path, 0660);
                        if (config == NULL)
                                return;
                        config_parse(config);
                }
        }

	data.config = config;
        data.save_all = save_all;
	g_hash_table_foreach(theme->modules, (GHFunc) module_save, &data);

	basename = g_path_get_basename(theme->path);
        /* always save the theme to ~/.irssi/ */
	path = g_strdup_printf("%s/%s", get_irssi_dir(), basename);
	ok = config_write(config, path, 0660) == 0;
	g_free(basename);

	printformat(NULL, NULL, MSGLEVEL_CLIENTNOTICE,
		    ok ? TXT_THEME_SAVED : TXT_THEME_SAVE_FAILED,
		    path, config_last_error(config));

	g_free(path);
	config_close(config);
}


void sig_dccget_connected(GET_DCC_REC *dcc)
{
	struct stat statbuf;
	char *fname, *tempfname, *str;
        int ret, ret_errno, temphandle, old_umask;

	if (!dcc->from_dccserver) {
		if (net_geterror(dcc->handle) != 0) {
			/* error connecting */
			signal_emit("dcc error connect", 1, dcc);
			dcc_destroy(DCC(dcc));
			return;
		}

		g_source_remove(dcc->tagconn);
		dcc->tagconn = -1;
	}

	g_free_not_null(dcc->file);
	dcc->file = dcc_get_download_path(dcc->arg);

	/* if some plugin wants to change the file name/path here.. */
	signal_emit("dcc get receive", 1, dcc);

	if (stat(dcc->file, &statbuf) == 0 &&
	    dcc->get_type == DCC_GET_RENAME) {
		/* file exists, rename.. */
		fname = dcc_get_rename_file(dcc->file);
		g_free(dcc->file);
		dcc->file = fname;
	}

	if (dcc->get_type != DCC_GET_RESUME) {
		int dcc_file_create_mode = octal2dec(settings_get_int("dcc_file_create_mode"));

		/* we want to overwrite the file, remove it here.
		   if it gets created after this, we'll fail. */
		unlink(dcc->file);

		/* just to make sure we won't run into race conditions
		   if download_path is in some global temp directory */
		tempfname = g_strconcat(dcc->file, ".XXXXXX", NULL);

                old_umask = umask(0077);
		temphandle = mkstemp(tempfname);
		umask(old_umask);

		if (temphandle == -1)
			ret = -1;
		else
			ret = fchmod(temphandle, dcc_file_create_mode);

		close(temphandle);

		if (ret != -1) {
			ret = link(tempfname, dcc->file);

			if (ret == -1 &&
			    /* Linux */
			    (errno == EPERM ||
			     /* FUSE */
			     errno == ENOSYS ||
			     /* BSD */
			     errno == EOPNOTSUPP)) {
				/* hard links aren't supported - some people
				   want to download stuff to FAT/NTFS/etc
				   partitions, so fallback to rename() */
				ret = rename(tempfname, dcc->file);
			}
		}

		/* if ret = 0, we're the file owner now */
		dcc->fhandle = ret == -1 ? -1 :
			open(dcc->file, O_WRONLY | O_TRUNC);

		/* close/remove the temp file */
		ret_errno = errno;
		unlink(tempfname);
		g_free(tempfname);

		if (dcc->fhandle == -1) {
			signal_emit("dcc error file create", 3,
				    dcc, dcc->file, g_strerror(ret_errno));
			dcc_destroy(DCC(dcc));
			return;
		}
	}

	dcc->starttime = time(NULL);
	if (dcc->size == 0) {
		dcc_close(DCC(dcc));
		return;
	}
	dcc->tagread = g_input_add(dcc->handle, G_INPUT_READ,
				   (GInputFunction) sig_dccget_receive, dcc);
	signal_emit("dcc connected", 1, dcc);

	if (dcc->from_dccserver) {
		str = g_strdup_printf("121 %s %d\n",
				      dcc->server ? dcc->server->nick : "??", 0);
		net_transmit(dcc->handle, str, strlen(str));
	}
}


static void sig_channel_joined(IRC_CHANNEL_REC *channel)
{
	if (!IS_IRC_CHANNEL(channel))
		return;

	if (!settings_get_bool("channel_sync"))
		return;

	/* Add channel to query lists */
	if (!channel->no_modes)
		query_add_channel(channel, CHANNEL_QUERY_MODE);
	if (g_hash_table_size(channel->nicks) <
	    settings_get_int("channel_max_who_sync"))
		query_add_channel(channel, CHANNEL_QUERY_WHO);
	if (!channel->no_modes)
		query_add_channel(channel, CHANNEL_QUERY_BMODE);

	query_check(channel->server);
}


static void copy_abstract_hash(char *key, char *value, GHashTable *dest)
{
	g_hash_table_insert(dest, g_strdup(key), g_strdup(value));
}


static void cmd_ctcp(const char *data, IRC_SERVER_REC *server)
{
	CHAT_DCC_REC *dcc;
	char *target, *ctcpcmd, *ctcpdata, *str;
	void *free_arg;

	g_return_if_fail(data != NULL);

	if (!cmd_get_params(data, &free_arg, 3 | PARAM_FLAG_GETREST,
			    &target, &ctcpcmd, &ctcpdata))
		return;
	if (*target == '\0' || *ctcpcmd == '\0')
		cmd_param_error(CMDERR_NOT_ENOUGH_PARAMS);

	if (*target != '=') {
		/* handle only DCC CTCPs */
		cmd_params_free(free_arg);
		return;
	}

	dcc = dcc_chat_find_id(target+1);
	if (dcc != NULL) {
		ascii_strup(ctcpcmd);

		str = g_strconcat(ctcpcmd, " ", ctcpdata, NULL);
		dcc_ctcp_message(server, dcc->nick, dcc, FALSE, str);
		g_free(str);
	}

	cmd_params_free(free_arg);
	signal_stop();
}


static void cmd_dcc_send(const char *data, IRC_SERVER_REC *server,
			 WI_ITEM_REC *item)
{
	const char *servertag;
	char *nick, *fileargs;
	void *free_arg;
	CHAT_DCC_REC *chat;
	GHashTable *optlist;
	int queue, mode, passive;

	if (!cmd_get_params(data, &free_arg, 2 | PARAM_FLAG_OPTIONS |
			    PARAM_FLAG_GETREST | PARAM_FLAG_STRIP_TRAILING_WS,
			    "dcc send", &optlist, &nick, &fileargs))
		return;

	chat = item_get_dcc(item);
	if (chat != NULL &&
	    (chat->mirc_ctcp || g_ascii_strcasecmp(nick, chat->nick) != 0))
		chat = NULL;

	if (IS_IRC_SERVER(server) && server->connected)
		servertag = server->tag;
	else if (chat != NULL)
		servertag = chat->servertag;
	else
		servertag = NULL;

	if (servertag == NULL && chat == NULL)
		cmd_param_error(CMDERR_NOT_CONNECTED);

	passive = g_hash_table_lookup(optlist, "passive") != NULL;

	if (g_hash_table_lookup(optlist, "rmhead") != NULL) {
		queue = dcc_queue_old(nick, servertag);
		if (queue != -1)
			dcc_queue_remove_head(queue);
	} else if (g_hash_table_lookup(optlist, "rmtail") != NULL) {
		queue = dcc_queue_old(nick, servertag);
		if (queue != -1)
			dcc_queue_remove_tail(queue);
	} else if (g_hash_table_lookup(optlist, "flush") != NULL) {
		queue = dcc_queue_old(nick, servertag);
		if (queue != -1)
			dcc_queue_free(queue);
	} else {
		if (g_hash_table_lookup(optlist, "append") != NULL)
			mode = DCC_QUEUE_APPEND;
		else if (g_hash_table_lookup(optlist, "prepend") != NULL)
			mode = DCC_QUEUE_PREPEND;
		else
			mode = DCC_QUEUE_NORMAL;

		if (*fileargs == '\0')
			cmd_param_error(CMDERR_NOT_ENOUGH_PARAMS);

		dcc_send_add(servertag, chat, nick, fileargs, mode, passive);
	}

	cmd_params_free(free_arg);
}


static void theme_real_destroy(THEME_REC *rec)
{
	g_hash_table_foreach(rec->abstracts, (GHFunc) theme_abstract_destroy, NULL);
	g_hash_table_destroy(rec->abstracts);
	g_hash_table_foreach(rec->modules, (GHFunc) theme_module_destroy, NULL);
	g_hash_table_destroy(rec->modules);

	g_slist_foreach(rec->replace_values, (GFunc) g_free, NULL);
	g_slist_free(rec->replace_values);

	g_free(rec->path);
	g_free(rec->name);
	g_free(rec);
}


static void themes_remove_module(const char *module)
{
        g_slist_foreach(themes, (GFunc) theme_remove_module, (void *) module);
}


static char *dcc_chat_get_new_id(const char *nick)
{
        char *id;
	int num;

        g_return_val_if_fail(nick != NULL, NULL);

	if (dcc_chat_find_id(nick) == NULL) {
		/* same as nick, good */
                return g_strdup(nick);
	}

	/* keep adding numbers after nick until some of them isn't found */
	for (num = 2;; num++) {
                id = g_strdup_printf("%s%d", nick, num);
		if (dcc_chat_find_id(id) == NULL)
                        return id;
                g_free(id);
	}
}


static void query_remove_all(IRC_CHANNEL_REC *channel)
{
	SERVER_QUERY_REC *rec;
	int n;

	rec = channel->server->chanqueries;
	if (rec == NULL) return;

	/* remove channel from query lists */
	for (n = 0; n < CHANNEL_QUERIES; n++)
		rec->queries[n] = g_slist_remove(rec->queries[n], channel);
	rec->current_queries = g_slist_remove(rec->current_queries, channel);

	if (!channel->server->disconnected)
		query_check(channel->server);
}


static void event_ping(IRC_SERVER_REC *server, const char *data)
{
	char *params, *origin, *target, *str;

	params = event_get_params(data, 2, &origin, &target);
	str = *target == '\0' ? g_strconcat("PONG :", origin, NULL) :
		g_strdup_printf("PONG %s :%s", target, origin);
	irc_send_cmd_now(server, str);
        g_free(str);
	g_free(params);
}


static void theme_abstract_destroy(char *key, char *value)
{
	g_free(key);
        g_free(value);
}


static void event_duplicate_channel(IRC_SERVER_REC *server, const char *data)
{
	CHANNEL_REC *chanrec;
	char *params, *channel, *p;

	g_return_if_fail(data != NULL);

	/* this new addition to ircd breaks completely with older
	   "standards", "nick Duplicate ::!!channel ...." */
	params = event_get_params(data, 3, NULL, NULL, &channel);
	p = strchr(channel, ' ');
	if (p != NULL) *p = '\0';

	if (channel[0] == '!') {
		chanrec = channel_find(SERVER(server),
				       channel+(channel[1] == '!'));
		if (chanrec != NULL && !chanrec->names_got) {
			chanrec->left = TRUE;
			channel_destroy(chanrec);
		}
	}

	g_free(params);
}


static void theme_get_modules(char *module, FORMAT_REC *formats, GSList **list)
{
	THEME_SEARCH_REC *rec;

	rec = g_new(THEME_SEARCH_REC, 1);
	rec->name = module;
	rec->short_name = strrchr(module, '/');
	if (rec->short_name != NULL)
		rec->short_name++; else rec->short_name = module;
	*list = g_slist_insert_sorted(*list, rec, (GCompareFunc) theme_search_equal);
}


void dcc_get_init(void)
{
        dcc_register_type("GET");
	settings_add_bool("dcc", "dcc_autorename", FALSE);
	settings_add_str("dcc", "dcc_download_path", "~");
	settings_add_int("dcc", "dcc_file_create_mode", 644);

	signal_add("dcc destroyed", (SIGNAL_FUNC) sig_dcc_destroyed);
	signal_add("ctcp msg dcc send", (SIGNAL_FUNC) ctcp_msg_dcc_send);
	command_bind("dcc get", NULL, (SIGNAL_FUNC) cmd_dcc_get);
}


ldns_fskipcs(FILE *fp, const char *s)
{
	ldns_fskipcs_l(fp, s, NULL);
}


ldns_str2rdf_wks(ldns_rdf **rd, const char *str)
{
	uint8_t *bitmap = NULL;
	uint8_t *data;
	int bm_len = 0;

	struct protoent *proto = NULL;
	struct servent *serv = NULL;
	int serv_port;

	ldns_buffer *str_buf;

	char *proto_str = NULL;
	char *lc_proto_str = NULL;
	char *token;
	char *lc_token;
	char *c;
	if(strlen(str) == 0)
		token = LDNS_XMALLOC(char, 50);
	else 	token = LDNS_XMALLOC(char, strlen(str)+2);
	if(!token) return LDNS_STATUS_MEM_ERR;

	str_buf = LDNS_MALLOC(ldns_buffer);
	if(!str_buf) {LDNS_FREE(token); return LDNS_STATUS_MEM_ERR;}
	ldns_buffer_new_frm_data(str_buf, (char *)str, strlen(str));
	if(ldns_buffer_status(str_buf) != LDNS_STATUS_OK) {
		LDNS_FREE(str_buf);
		LDNS_FREE(token);
		return LDNS_STATUS_MEM_ERR;
	}

	while(ldns_bget_token(str_buf, token, "\t\n ", strlen(str)) > 0) {
		if (!proto_str) {
			proto_str = strdup(token);
			lc_proto_str = strdup(token);
			for (c = lc_proto_str; *c; c++) {
				*c = tolower(*c);
			}
			if (!proto_str || !lc_proto_str) {
				free(proto_str);
				free(lc_proto_str);
				LDNS_FREE(bitmap);
				LDNS_FREE(token);
	                        ldns_buffer_free(str_buf);
				return LDNS_STATUS_INVALID_STR;
			}
		} else {
			serv = getservbyname(token, proto_str);
			if (!serv) {
				serv = getservbyname(token, lc_proto_str);
			}
			if (!serv && (lc_token = strdup(token))) {
				for (c = lc_token; *c; c++) {
					*c = tolower(*c);
				}
				serv = getservbyname(lc_token, proto_str);
				if (!serv) {
					serv = getservbyname(lc_token, lc_proto_str);
				}
				free(lc_token);
			}
			if (serv) {
				serv_port = (int) ntohs((uint16_t) serv->s_port);
			} else {
				serv_port = atoi(token);
			}
			if (serv_port / 8 >= bm_len) {
				uint8_t *b2 = LDNS_XREALLOC(bitmap, uint8_t, (serv_port / 8) + 1);
                                if(!b2) {
					LDNS_FREE(bitmap);
				        LDNS_FREE(token);
	                                ldns_buffer_free(str_buf);
				        free(proto_str);
				        free(lc_proto_str);
				        return LDNS_STATUS_INVALID_STR;
                                }
				bitmap = b2;
				/* set to zero to be sure */
				for (; bm_len <= serv_port / 8; bm_len++) {
					bitmap[bm_len] = 0;
				}
			}
			ldns_set_bit(bitmap + (serv_port / 8), 7 - (serv_port % 8), true);
		}
	}

	if (!proto_str || !bitmap) {
		LDNS_FREE(bitmap);
		LDNS_FREE(token);
	        ldns_buffer_free(str_buf);
	        free(proto_str);
	        free(lc_proto_str);
		return LDNS_STATUS_INVALID_STR;
	}

	data = LDNS_XMALLOC(uint8_t, bm_len + 1);
        if(!data) {
	        LDNS_FREE(token);
	        ldns_buffer_free(str_buf);
	        LDNS_FREE(bitmap);
	        free(proto_str);
	        free(lc_proto_str);
	        return LDNS_STATUS_INVALID_STR;
        }
    if (proto_str)
		proto = getprotobyname(proto_str);
    	if (!proto) {
		proto = getprotobyname(lc_proto_str);
	}
	if (proto) {
		data[0] = (uint8_t) proto->p_proto;
	} else if (proto_str) {
		data[0] = (uint8_t) atoi(proto_str);
	}
	memcpy(data + 1, bitmap, (size_t) bm_len);

	*rd = ldns_rdf_new_frm_data(LDNS_RDF_TYPE_WKS, (uint16_t) (bm_len + 1), data);

	LDNS_FREE(data);
	LDNS_FREE(token);
	ldns_buffer_free(str_buf);
	LDNS_FREE(bitmap);
	free(proto_str);
	free(lc_proto_str);
#ifdef HAVE_ENDSERVENT
	endservent();
#endif
#ifdef HAVE_ENDPROTOENT
	endprotoent();
#endif

	if(!*rd) return LDNS_STATUS_MEM_ERR;

	return LDNS_STATUS_OK;
}


ldns_str2rdf_nsec3_salt(ldns_rdf **rd, const char *salt_str)
{
	uint8_t salt_length;
	int c;
	int salt_length_str;

	uint8_t *salt;
	uint8_t *data;
	if(rd == NULL) {
		return LDNS_STATUS_NULL;
	}

	salt_length_str = (int)strlen(salt_str);
	if (salt_length_str == 1 && salt_str[0] == '-') {
		salt_length_str = 0;
	} else if (salt_length_str % 2 != 0) {
		return LDNS_STATUS_INVALID_HEX;
	}
	if (salt_length_str > 512) {
		return LDNS_STATUS_INVALID_HEX;
	}

	salt = LDNS_XMALLOC(uint8_t, salt_length_str / 2);
        if(!salt) {
                return LDNS_STATUS_MEM_ERR;
        }
	for (c = 0; c < salt_length_str; c += 2) {
		if (isxdigit((int) salt_str[c]) && isxdigit((int) salt_str[c+1])) {
			salt[c/2] = (uint8_t) ldns_hexdigit_to_int(salt_str[c]) * 16 +
					  ldns_hexdigit_to_int(salt_str[c+1]);
		} else {
			LDNS_FREE(salt);
			return LDNS_STATUS_INVALID_HEX;
		}
	}
	salt_length = (uint8_t) (salt_length_str / 2);

	data = LDNS_XMALLOC(uint8_t, 1 + salt_length);
        if(!data) {
	        LDNS_FREE(salt);
                return LDNS_STATUS_MEM_ERR;
        }
	data[0] = salt_length;
	memcpy(&data[1], salt, salt_length);
	*rd = ldns_rdf_new_frm_data(LDNS_RDF_TYPE_NSEC3_SALT, 1 + salt_length, data);
	LDNS_FREE(data);
	LDNS_FREE(salt);

	return *rd?LDNS_STATUS_OK:LDNS_STATUS_MEM_ERR;
}


ldns_str2rdf_atma(ldns_rdf **rd, const char *str)
{
    size_t len, i;
    char* atma_str = (char*) str;
	ldns_status status;

	/* just a hex string with optional dots? */
	len = strlen(str);
	for (i=0; i < len; i++) {
		if (atma_str[i] == '.')
			atma_str[i] = ' ';
	}
	status = ldns_str2rdf_hex(rd, str);
    if (status != LDNS_STATUS_OK) {
		; /* probably in e.164 format than */
	}
	return status;
}


ldns_str2rdf_int8(ldns_rdf **rd, const char *bytestr)
{
	char *end;
	uint8_t *r = NULL;

	r = LDNS_MALLOC(uint8_t);
        if(!r) return LDNS_STATUS_MEM_ERR;

	*r = (uint8_t)strtol((char*)bytestr, &end, 10);

        if(*end != 0) {
		LDNS_FREE(r);
		return LDNS_STATUS_ERR;
        } else {
		*rd = ldns_rdf_new_frm_data(
			LDNS_RDF_TYPE_INT8, sizeof(uint8_t), r);
		LDNS_FREE(r);
	        return *rd?LDNS_STATUS_OK:LDNS_STATUS_MEM_ERR;
        }
}


ldns_wire2pkt(ldns_pkt **packet_p, const uint8_t *wire, size_t max)
{
	size_t pos = 0;
	uint16_t i;
	ldns_rr *rr;
	ldns_pkt *packet = ldns_pkt_new();
	ldns_status status = LDNS_STATUS_OK;
	int have_edns = 0;

	uint8_t data[4];

	status = ldns_wire2pkt_hdr(packet, wire, max, &pos);
	LDNS_STATUS_CHECK_GOTO(status, status_error);

	for (i = 0; i < ldns_pkt_qdcount(packet); i++) {

		status = ldns_wire2rr(&rr, wire, max, &pos, LDNS_SECTION_QUESTION);
		if (status == LDNS_STATUS_PACKET_OVERFLOW) {
			status = LDNS_STATUS_WIRE_INCOMPLETE_QUESTION;
		}
		LDNS_STATUS_CHECK_GOTO(status, status_error);
		if (!ldns_rr_list_push_rr(ldns_pkt_question(packet), rr)) {
			ldns_pkt_free(packet);
			return LDNS_STATUS_INTERNAL_ERR;
		}
	}
	for (i = 0; i < ldns_pkt_ancount(packet); i++) {
		status = ldns_wire2rr(&rr, wire, max, &pos, LDNS_SECTION_ANSWER);
		if (status == LDNS_STATUS_PACKET_OVERFLOW) {
			status = LDNS_STATUS_WIRE_INCOMPLETE_ANSWER;
		}
		LDNS_STATUS_CHECK_GOTO(status, status_error);
		if (!ldns_rr_list_push_rr(ldns_pkt_answer(packet), rr)) {
			ldns_pkt_free(packet);
			return LDNS_STATUS_INTERNAL_ERR;
		}
	}
	for (i = 0; i < ldns_pkt_nscount(packet); i++) {
		status = ldns_wire2rr(&rr, wire, max, &pos, LDNS_SECTION_AUTHORITY);
		if (status == LDNS_STATUS_PACKET_OVERFLOW) {
			status = LDNS_STATUS_WIRE_INCOMPLETE_AUTHORITY;
		}
		LDNS_STATUS_CHECK_GOTO(status, status_error);
		if (!ldns_rr_list_push_rr(ldns_pkt_authority(packet), rr)) {
			ldns_pkt_free(packet);
			return LDNS_STATUS_INTERNAL_ERR;
		}
	}
	for (i = 0; i < ldns_pkt_arcount(packet); i++) {
		status = ldns_wire2rr(&rr, wire, max, &pos, LDNS_SECTION_ADDITIONAL);
		if (status == LDNS_STATUS_PACKET_OVERFLOW) {
			status = LDNS_STATUS_WIRE_INCOMPLETE_ADDITIONAL;
		}
		LDNS_STATUS_CHECK_GOTO(status, status_error);

		if (ldns_rr_get_type(rr) == LDNS_RR_TYPE_OPT) {
			ldns_pkt_set_edns_udp_size(packet, ldns_rr_get_class(rr));
			ldns_write_uint32(data, ldns_rr_ttl(rr));
			ldns_pkt_set_edns_extended_rcode(packet, data[0]);
			ldns_pkt_set_edns_version(packet, data[1]);
			ldns_pkt_set_edns_z(packet, ldns_read_uint16(&data[2]));
			/* edns might not have rdfs */
			if (ldns_rr_rdf(rr, 0)) {
				ldns_pkt_set_edns_data(packet, ldns_rdf_clone(ldns_rr_rdf(rr, 0)));
			}
			ldns_rr_free(rr);
			have_edns += 1;
		} else if (ldns_rr_get_type(rr) == LDNS_RR_TYPE_TSIG) {
			ldns_pkt_set_tsig(packet, rr);
			ldns_pkt_set_arcount(packet, ldns_pkt_arcount(packet) - 1);
		} else if (!ldns_rr_list_push_rr(ldns_pkt_additional(packet), rr)) {
			ldns_pkt_free(packet);
			return LDNS_STATUS_INTERNAL_ERR;
		}
	}
	ldns_pkt_set_size(packet, max);
	if(have_edns)
		ldns_pkt_set_arcount(packet, ldns_pkt_arcount(packet)
                        - have_edns);

	*packet_p = packet;
	return status;

status_error:
	ldns_pkt_free(packet);
	return status;
}


ldns_rdf_type_maybe_quoted(ldns_rdf_type rdf_type)
{
	return  rdf_type == LDNS_RDF_TYPE_STR ||
		rdf_type == LDNS_RDF_TYPE_LONG_STR ||
		rdf_type == LDNS_RDF_TYPE_MULTI_STR;
}