谢太数据库

Linkedin/Glu Deployment Automation Platform

2011-11-16T07:25:00.001+08:00

Glu自动化部署平台

What is glu?
glu is a free/open source deployment and monitoring automation platform.

What problems does glu solve?
glu is solving the following problems:
deploy (and monitor) applications to an arbitrary large set of nodes:
efficiently
with minimum/no human interaction
securely
in a reproducible manner
ensure consistency over time (prevent drifting)
detect and troubleshoot quickly when problems arise

How does it work?
glu takes a very declarative approach, in which you describe/model what you want, and glu can then:
compute the set of actions to deploy/upgrade your applications
ensure that it remains consistent over time
detect and alert you when there is a mismatch

Glu has 3 major components. Agents, Orchestration Engine, and ZooKeeper.

glu vs puppet
puppet is very good at configuring the infrastructure of a machine (users, groups, packages, etc...)

=> static/stable does not change often

glu is very good at provisioning dynamic applications on an ensemble of machines (the system)

=> changes often, real-time failure detection (monitoring), "bounce", etc...

glu
Glu Tech Talk 201107
Glu-ing the Last Mile by Ken Sipe.
Continuous Deployment at outbrain
The future of glu - glu in the cloud
Monitoring a Wild Beast

bash merge sort

2011-09-01T14:22:00.000+08:00

bash merge sort

bash归并排序


#!/bin/bash
rest() { shift; echo $@; }
length() { echo $#; }
first() { echo $1; }
merge()
{
  declare left=$1 right=$2 result=
  declare length_left=$(length $left)
  declare length_right=$(length $right)
  while (( length_left>0 || length_right>0 )); do
    if (( length_left>0 && length_right>0 )); then
      if [ $(first $left) -le $(first $right) ]; then
        result="$result $(first $left)"
        left=$(rest $left)
      else
        result="$result $(first $right)"
        right=$(rest $right)
      fi
    elif [ $(length $left) -gt 0 ]; then
      result="$result $(first $left)"
      left=$(rest $left)
    elif [ $(length $right) -gt 0 ]; then
      result="$result $(first $right)"
      right=$(rest $right)
    fi
    length_left=$(length $left)
    length_right=$(length $right)
  done
  echo $result
}
merge_sort()
{
  [ $# -le 1 ] && { echo $@; return; }
  declare -i middle=$(($#/2+1))
  declare left= right= result=
  for ((i=1;i<middle;i++)); do
    left="$left ${!i}"
  done
  for ((i=middle;i<=$#;i++)); do
    right="$right ${!i}"
  done
  left=$(merge_sort $left)
  right=$(merge_sort $right)
  result=$(merge "$left" "$right")
  echo $result
}
merge_sort $@

执行：


# ./merge_sort.sh $(for in in {1..50}; do echo $((RANDOM%200-100)); done)
-99 -89 -88 -84 -83 -83 -81 -76 -75 -70 -70 -70 -54 -53 -51 -51 -49 -49 -40 -40 -30 -27 -24 -19 -15 -13 -12 -12 -8 -2 -2 -2 2 4 10 17 20 24 28 34 39 39 40 42 45 48 48 49 62 83

算法参考：Merge sort

-fin-

bash quicksort

2011-08-30T19:30:00.002+08:00

bash quicksort

用bash实现快速排序
参考Quicksort算法实现，如下：


function swap()
{
  declare -r _array_name=$1
  declare -r _i1=$2
  declare -r _i2=$3
  eval declare -r _tmp=\${${_array_name}[$_i1]}
  eval ${_array_name}[$_i1]=\${${_array_name}[$_i2]}
  eval ${_array_name}[$_i2]=\$_tmp
}
function partition()
{
  declare -r _array_name=$1
  declare -r _left=$2
  declare -r _right=$3
  declare -r _pivot_index=${4:-$_left}
  eval declare -r _pivot_value=\${${_array_name}[$_pivot_index]}
  swap $_array_name $_pivot_index $_right
  declare _store_index=$_left
  for ((_i=_left; _i<_right; _i++)); do
    eval declare _v=\${${_array_name}[$_i]}
    if (( _v < _pivot_value )); then
      swap $_array_name $_i $_store_index
      ((_store_index++))
    fi
  done
  swap $_array_name $_right $_store_index
  #echo "store_index=$_store_index"
  return $_store_index
}
function quick_sort()
{  
  declare -r _array_name=$1
  declare -r _left=$2
  declare -r _right=$3
  #declare -r _tmpfile=${TMPDIR:-/tmp}/qs_tmpfile.$$
  #((_right-_left>256)) && { echo "do not support more than 256 elements in array." >&2; return 1; }
  if ((_left<_right)); then
    declare _pivot_index=$_left
    #partition $_array_name $_left $_right $_pivot_index >$_tmpfile
    #declare _pivot_new_index=$(sed -n '/^store_index=/s/.*=//p' $_tmpfile)
    partition $_array_name $_left $_right $_pivot_index
    declare _pivot_new_index=$?
    quick_sort $_array_name $_left $((_pivot_new_index-1))
    quick_sort $_array_name $((_pivot_new_index+1)) $_right
  fi
  #[ -f $_tmpfile ] && /bin/rm -f $_tmpfile
}

稍微简化一下，改成：


function quick_sort2()
{  
  swap()
  {
    eval declare -r _tmp=\${${_array_name}[$1]}
    eval ${_array_name}[$1]=\${${_array_name}[$2]}
    eval ${_array_name}[$2]=\$_tmp
  }
  declare -r _array_name=$1
  declare -r _left=$2
  declare -r _right=$3
  if ((_left<_right)); then
    declare -r _pivot_index=$_left
    #declare -r _pivot_index=$(( ($_right+$_left)/2 ))
    eval declare _pivot_value=\${${_array_name}[$_pivot_index]}
    swap $_pivot_index $_right
    declare _store_index=$_left
    for ((_i=_left; _i<_right; _i++)); do
      eval declare _v=\${${_array_name}[$_i]}
      if (( _v < _pivot_value )); then
        swap $_i $_store_index
        ((_store_index++))
      fi
    done
    swap $_right $_store_index
    quick_sort2 $_array_name $_left $((_store_index-1))
    quick_sort2 $_array_name $((_store_index+1)) $_right
  fi
}

运行结果


# declare -a a=($(for in in {1..50}; do echo $((RANDOM%200-100)); done))
# quick_sort2 a 0 $((${#a[@]}-1))
# echo "${a[@]}"
-99 -97 -96 -94 -90 -82 -77 -76 -72 -70 -66 -66 -66 -63 -59 -53 -51 -50 -50 -48 -46 -45 -38 -34 -33 -30 -22 -21 -21 -20 -18 -10 0 2 2 3 9 24 26 26 44 46 47 47 62 66 80 95 99 99
#

其实网上有个更简单的办法，Tweetable Quicksort by john，一行脚本就可搞定了。


q(){ l=;g=;[ $# -lt 2 ]&&echo $@||(for n in ${@:2};do [ $n -gt $1 ]&&g="$g$n "||l="$l$n ";done;echo `q $l` $1 `q $g`;)}

运行


# q $(for in in {1..50}; do echo $((RANDOM%200-100)); done)
-100 -96 -92 -91 -80 -75 -71 -66 -54 -52 -44 -43 -40 -33 -32 -29 -28 -23 -20 -12 -12 1 16 22 22 27 27 33 33 35 38 39 41 44 45 48 50 55 58 61 62 66 69 72 83 83 85 94 97 98

外部链接：
一个Quicksort究竟可以写到多么短
快速排序（Quicksort）的Javascript实现
数学之美番外篇：快排为什么那样快

又看到一个SLEEP排序，很有趣
Genius sorting algorithm: Sleep sort


#!/bin/bash
function f() {
    sleep "$1"
    echo "$1"
}
while [ -n "$1" ]
do
    f "$1" &
    shift
done
wait

example usage:
./sleepsort.bash 5 3 6 3 6 3 1 4 7

-fin-

Getting MySQL process list with GDB

2011-08-17T21:34:00.009+08:00

用GDB调试MySQL，显示线程列表

define helper functions

cat >mysql.gdb <<'EOF'
#set interactive-mode off
set confirm off
dont-repeat
define show_processlist
  #argc is not available in older version gdb
  #if $argc==1
  #  set $p_thread_id=(long long)$arg0
  #else
    set $p_thread_id=(long long)-1
  #end
  set $current=((base_ilist)threads).first
  set $last=&(((base_ilist)threads).last)
  printf "Id\tUser\tHost\tDb\tCommand\tTime\tState\tInfo\n"
  while $current != $last
    set $tmp=(class THD *)$current
    set $tmp_sctx=$tmp->security_ctx
    if ( $p_thread_id == -1 || $p_thread_id == $tmp->thread_id )
      set $id=$tmp->thread_id
      set $user=(char*)($tmp_sctx->user?$tmp_sctx->user:($tmp->system_thread?"system user":"unauthenticated user"))
      set $host=$tmp_sctx->host_or_ip
      set $db=($tmp->db)?$tmp->db:"NULL"
      set $command=(int)($tmp->command)
      #set $proc_info=(char*)(($tmp->killed==THD::KILL_CONNECTION)?"Killed":0) #5.1
      set $proc_info=(char*)($tmp->killed?"Killed":0)
      set $state_info=(char*) ($tmp->locked ? "Locked" : $tmp->net.reading_or_writing ? ($tmp->net.reading_or_writing == 2 ? "Writing to net" : $command == COM_SLEEP ? "" : "Reading from net") : $tmp->proc_info ? $tmp->proc_info : ($tmp->mysys_var && $tmp->mysys_var->current_cond) ? "Waiting on cond" : "NULL")
      set $start_time=$tmp->start_time
      #set $now=my_time(0) #5.1
      set $now=time((time_t*) 0)
      set $query=(char*)($tmp->query_string.str?$tmp->query_string.str:"")
      printf "%u\t%s\t%s\t%s\t%s\t%u\t%s\t%.100s\n",$id,$user,$host,$db,command_name[$command].str,$now-$start_time,$state_info,$query
    end
    set $current=$current->next
  end
end
document show_processlist
  Prints MySQL process list.
  Syntax: show_processlist [thread id]
  Examples:
    show_processlist
    show_processlist 123
    show_processlist -1
end
define kill_thread
  #if $argc >= 1
    set $p_thread_id=(unsigned long long)$arg0
    #set $p_only_kill_query=(char *)(($argc >= 2)?$arg1:"connection")
    set $p_only_kill_query=(char *)$arg1
    set $only_kill_query=(strcmp($p_only_kill_query,(char *)"query")==0)?1:0
    set $current=((base_ilist)threads).first
    set $last=&(((base_ilist)threads).last)
    while $current != $last
      set $tmp=(class THD *)$current
      if $p_thread_id == $tmp->thread_id
        set $id=$tmp->thread_id
        printf "kill thread id %d %s\n", $id, $only_kill_query?"only query":""
        #call kill_one_thread($tmp, $id, $only_kill_query)
        call sql_kill($tmp, $id, $only_kill_query)
        loop_break
      end
      set $current=$current->next
    end
  #else
  #  help kill_thread
  #end
end
document kill_thread
  Kills MySQL thread.
  Syntax: kill_thread <thread id> <query|connection>
  Examples:
    kill_thread 123
    kill_thread 123 "connection"
    kill_thread 321 "query"
end
EOF

show processlist

mysql_port=8217
# older version gdb does not support '-ex' option
#gdb -batch -silent -p $(pgrep -f "/mysqld .*$mysql_port"|head -n1) -x mysql.gdb -ex 'show_processlist'
gdb -silent -p $(pgrep -f "/mysqld .*$mysql_port"|head -n1) -x mysql.gdb <<'EOF' |\
sed -n '/^=====begin=====$/,/=====end=====/p'|sed '1d;$d'
set prompt
set pagination off
printf "\n\n"
printf "=====begin=====\n"
show_processlist
printf "=====end=====\n"
EOF

kill thread

thread_id=223
gdb -silent -p $(pgrep -f "/mysqld .*$mysql_port"|head -n1) -x mysql.gdb <<EOF
kill_thread $thread_id "connect"
EOF

MySQL General Query Log

2011-08-16T01:13:00.000+08:00

MySQL General Query Log

==参考文档==
The General Query Log: http://dev.mysql.com/doc/refman/5.1/en/query-log.html
Selecting General Query and Slow Query Log Output Destinations: http://dev.mysql.com/doc/refman/5.1/en/log-destinations.html
mysql-5.1.52.tar.gz mysql-5.1.52.tar.gz

==系统变量==
MySQL 5.1.29+
* general_log 普通查询日志开关
* general_log_file 普通查询日志文件名
* log_output 日志输出方式
* sql_log_off 会话级普通查询日志开关

==代码分析==

* 日志相关操作的代码文件是：sql/log.cc和sql/log.h

* 记录日志函数只有两个：general_log_write和general_log_print
** 两者区别主要是输入的参数不一样，general_log_print是可变参数的，general_log_write参数数量是固定的

* 函数内判断操作是否需要记录日志。判断依据是：


  if (*general_log_handler_list && (what_to_log & (1L << (uint) command)))

what_to_log变量在sql/mysqld.cc初始化为二进制的15个1（COM_TIME等于15）


what_to_log= ~ (1L << (uint) COM_TIME);

因此判断条件(what_to_log & (1L << (uint) command))总是真的，默认所有操作都被记录。好像没有手工设置what_to_log的地方？

* MySQL其它程序调general_log_write或general_log_print函数写日志，比如当建立连接、退出、查询等操作时。经查看代码，有以下地方记录日志：
** sql/sql_connect.cc
*** check_user(connect和change user都调用此函数)
**** ER_NOT_SUPPORTED_AUTH_MODE和ER_SERVER_IS_IN_SECURE_AUTH_MODE,ER_ACCESS_DENIED_ERROR记录日志：general_log_print COM_CONNECT
**** 认证通过后也记录日志：general_log_print command(COM_CONNECT或COM_CHANGE_USER)
** sql/sql_parse.cc
*** COM_INIT_DB
**** use命令执行成功后记录日志 general_log_write
*** COM_QUERY general_log_write
*** COM_FIELD_LIST general_log_print
*** COM_QUIT general_log_print
*** COM_CREATE_DB general_log_print
*** COM_DROP_DB general_log_write
*** COM_BINLOG_DUMP general_log_print
*** COM_REFRESH general_log_print
*** COM_SHUTDOWN general_log_print
*** COM_STATISTICS general_log_print
*** COM_PROCESS_INFO general_log_print
*** COM_DEBUG general_log_print
** sql/sql_prepare.cc
*** mysqld_stmt_reset general_log_print (COM_STMT_RESET)
*** mysqld_stmt_close general_log_print (COM_STMT_CLOSE)
*** mysql_stmt_get_longdata general_log_print (COM_STMT_SEND_LONG_DATA)
*** Prepared_statement::prepare
**** 存储过程内语句不记录日志 general_log_write COM_STMT_PREPARE
*** Prepared_statement::execute
**** 存储过程内语句不记录日志 general_log_write COM_STMT_EXECUTE
** sql/sp_head.cc
*** sp_instr_stmt::execute
**** general_log_write COM_QUERY
** sql/sql_show.cc
*** mysqld_show_create_db
*** 当权限不够时记录日志 general_log_print COM_INIT_DB
** sql/log_event.cc
*** Query_log_event::do_apply_event
**** If the query was not ignored, it is printed to the general log general_log_write COM_QUERY
*** Xid_log_event::do_apply_event
**** For a slave Xid_log_event is COMMIT general_log_print COM_QUERY
** sql/slave.cc
*** connect_to_master
**** slave重连接master时记录日志 general_log_print COM_CONNECT_OUT
** sql/sql_db.cc
*** mysql_change_db
**** ER_DBACCESS_DENIED_ERROR无权限访问数据库 general_log_print COM_INIT_DB

* 这两个函数内分别调用了LOGGER::general_log_write和LOGGER::general_log_print方法
* LOGGER::general_log_print又调用了LOGGER::general_log_write
* LOGGER::general_log_write根据log_output变量设置的日志输出方式，可能有多个，再调用当前日志句柄中的log_general方法

* Log_to_csv_event_handler::log_general

...

==格式==

===表===
# 当前时间
# 用户名和主机名
## 格式是：priv_user[user] @ host[ip]
# 线程标识
# 命令类型
# 查询语句

===文件===
# 当前时间
# 线程标识
# 命令类型
# 查询语句

==DML操作==
不支持DML操作

===delete===
不能delete日志表


mysql> delete from mysql.general_log;
ERROR 1556 (HY000): You can't use locks with log tables.

避免报错，可rename表名后再操作


SET @old_log_state = @@global.general_log;
SET GLOBAL general_log = 'OFF';
rename table general_log to general_log_tmp;
delete from general_log_tmp;
rename table general_log_tmp to general_log;
SET GLOBAL general_log = @old_log_state;

====相关代码====
sql/lock.cc


int mysql_lock_tables_check(THD *thd, TABLE **tables, uint count, uint flags)
...
      if ((t->reginfo.lock_type >= TL_READ_NO_INSERT)
          || (thd->lex->sql_command == SQLCOM_LOCK_TABLES))
      {
        my_error(ER_CANT_LOCK_LOG_TABLE, MYF(0));
        DBUG_RETURN(1);
      }
...

==DDL操作==
一些DDL操作不被支持

===drop===
不支持在线drop操作


mysql> drop table mysql.general_log;
ERROR 1580 (HY000): You cannot 'DROP' a log table if logging is enabled

====相关代码====
sql/sql_table.cc


int mysql_rm_table_part2(THD *thd, TABLE_LIST *tables, bool if_exists,
...
  /* Disable drop of enabled log tables, must be done before name locking */
  for (table= tables; table; table= table->next_local)
  {
    if (check_if_log_table(table->db_length, table->db,
                           table->table_name_length, table->table_name, 1))
    {
      my_error(ER_BAD_LOG_STATEMENT, MYF(0), "DROP");
      pthread_mutex_unlock(&LOCK_open);
      DBUG_RETURN(1);
    }
  }
...

===truncate===
支持在线truncate

==存储引擎==
默认用CSV存储引擎。只支持CSV或MyISAM


mysql> set global general_log=0;
Query OK, 0 rows affected (0.00 sec)

mysql> alter table mysql.general_log engine=innodb;
ERROR 1579 (HY000): This storage engine cannot be used for log tables"

修改为Innodb存储引擎：


SET @old_log_state = @@global.general_log;
SET GLOBAL general_log = 'OFF';
use mysql
drop table if exists general_log_new;
create table general_log_new like general_log;
alter table general_log_new engine=innodb;
rename table general_log to general_log_old, general_log_new to general_log;
show create table general_log\G
SET GLOBAL general_log = @old_log_state;

====相关代码====
sql/sql_table.cc


bool mysql_alter_table(THD *thd,char *new_db, char *new_name,
...
  /*
    Check if we attempt to alter mysql.slow_log or
    mysql.general_log table and return an error if
    it is the case.
    TODO: this design is obsolete and will be removed.
  */
  if (table_list && table_list->db && table_list->table_name)
  {
    int table_kind= 0;

    table_kind= check_if_log_table(table_list->db_length, table_list->db,
                                   table_list->table_name_length,
                                   table_list->table_name, 0);

    if (table_kind)
    {
      /* Disable alter of enabled log tables */
      if (logger.is_log_table_enabled(table_kind))
      {
        my_error(ER_BAD_LOG_STATEMENT, MYF(0), "ALTER");
        DBUG_RETURN(TRUE);
      }

      /* Disable alter of log tables to unsupported engine */
      if ((create_info->used_fields & HA_CREATE_USED_ENGINE) &&
          (!create_info->db_type || /* unknown engine */
           !(create_info->db_type->flags & HTON_SUPPORT_LOG_TABLES)))
      {
        my_error(ER_UNSUPORTED_LOG_ENGINE, MYF(0));
        DBUG_RETURN(TRUE);
      }

#ifdef WITH_PARTITION_STORAGE_ENGINE
      if (alter_info->flags & ALTER_PARTITION)
      {
        my_error(ER_WRONG_USAGE, MYF(0), "PARTITION", "log table");
        DBUG_RETURN(TRUE);
      }
#endif
    }
  }
...

HTON_SUPPORT_LOG_TABLES 表示该存储引擎是否支持日志表


$ grep -r HTON_SUPPORT_LOG_TABLES *
sql/sql_table.cc:           !(create_info->db_type->flags & HTON_SUPPORT_LOG_TABLES)))
sql/handler.h:#define HTON_SUPPORT_LOG_TABLES      (1 << 7) //Engine supports log tables
storage/myisam/ha_myisam.cc:  myisam_hton->flags= HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES;
storage/csv/ha_tina.cc:  tina_hton->flags= (HTON_CAN_RECREATE | HTON_SUPPORT_LOG_TABLES |

-fin-

MySQL Proxy RW Splitting

2011-08-16T01:08:00.000+08:00

MySQL Proxy RW Splitting

=rw-splitting.lua=

==说明==
* MySQL Proxy 0.8.1版本的读写分离脚本
* 下载地址
mysql-proxy-0.8.1-linux-glibc2.3-x86-64bit.tar.gz
* 参考文档
** 读写分离脚本说明
MySQL Proxy RW Splitting: http://forge.mysql.com/wiki/MySQL_Proxy_RW_Splitting
MySQL Proxy learns R/W Splitting: http://jan.kneschke.de/projects/mysql/mysql-proxy-learns-r-w-splitting
MySQL Proxy: more R/W splitting: http://jan.kneschke.de/projects/mysql/mysql-proxy-more-r-w-splitting/
** Lua开发
Lua 5.1 Reference Manual: http://www.lua.org/manual/5.1/manual.html
Programming in Lua (first edition): http://www.lua.org/pil/
** MySQL Proxy脚本开发
MySQL Proxy Scripting: http://dev.mysql.com/doc/refman/5.5/en/mysql-proxy-scripting.html
MySQL Proxy Snippet: http://forge.mysql.com/tools/search.php?t=tag&k=mysqlproxy
MySQL Proxy Dev: http://forge.mysql.com/wiki/MySQL_Proxy_Dev
Advanced LUA Scripting: http://forge.mysql.com/wiki/Advanced_LUA_Scripting
ProxyCookbook: http://forge.mysql.com/wiki/ProxyCookbook
http://blog.suncrescent.net/2008/05/partial-mysql-proxy-api-doc/

==代码==
===1-20行===


  1 --[[ $%BEGINLICENSE%$
  2  Copyright (c) 2007, 2009, Oracle and/or its affiliates. All rights reserved.
  3
  4  This program is free software; you can redistribute it and/or
  5  modify it under the terms of the GNU General Public License as
  6  published by the Free Software Foundation; version 2 of the
  7  License.
  8
  9  This program is distributed in the hope that it will be useful,
 10  but WITHOUT ANY WARRANTY; without even the implied warranty of
 11  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 12  GNU General Public License for more details.
 13
 14  You should have received a copy of the GNU General Public License
 15  along with this program; if not, write to the Free Software
 16  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 17  02110-1301  USA
 18
 19  $%ENDLICENSE%$ --]]
 20

* 声明版权信息
* lua脚本中 "--\[\[" 和 "--\]\]" 之间表示多行注释块

===21-29行===


 21 ---
 22 -- a flexible statement based load balancer with connection pooling
 23 --
 24 -- * build a connection pool of min_idle_connections for each backend and maintain
 25 --   its size
 26 -- *
 27 --
 28 --
 29

* 脚本说明
* lua脚本中 "--" 之后表示单行注释信息

===30-34行===


 30 local commands    = require("proxy.commands")
 31 local tokenizer   = require("proxy.tokenizer")
 32 local lb          = require("proxy.balance")
 33 local auto_config = require("proxy.auto-config")
 34

* require加载MySQL Proxy模块
* 模块文件所在缺省目录是$LUA_HOME/lib/mysql-proxy/lua/proxy/
* 各模块说明如下：
** commands.lua：解析网络包中的MySQL命令
*** 参考 http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#Command_Packet_.28Overview.29
** tokenizer.lua：加载模块mysql.tokenizer；格式化SQL语句。没看懂
** balance.lua：负载均衡
*** function idle_failsafe_rw：获取一个空闲的可读写的连接（可读写的主库或备库）
*** function idle_ro：获取一个空闲的只读的连接（只读备库）
** auto-config.lua：加载模块lpeg，即"Parsing Expression Grammars For Lua"；管理配置参数
*** 语法
**** proxy show config 显示配置
**** proxy set global xxx.yyy=zzz 设置配置
**** proxy save config into "xxx" 保存配置
**** proxy load config from "xxx" 加载配置
*** 参考
**** http://www.inf.puc-rio.br/~roberto/lpeg/
**** http://www.gammon.com.au/scripts/doc.php?general=lua_lpeg
*** 问题
**** [[#proxy set global 报错 ERROR 1064 (42000): You have an error in your SQL syntax]]
**** [[#proxy set global 报错 ERROR 1105 (07000): (lua) proxy.response.errmsg is nil]]
**** [[#proxy save config 保存配置文件大小等于0]]
**** [[#proxy save config 保存配置文件名带双引号]]

===35-46行===


 35 --- config
 36 --
 37 -- connection pool
 38 if not proxy.global.config.rwsplit then
 39         proxy.global.config.rwsplit = {
 40                 min_idle_connections = 4,
 41                 max_idle_connections = 8,
 42
 43                 is_debug = false
 44         }
 45 end
 46

* 设置读写分离的缺省配置参数

===47-55行===


 47 ---
 48 -- read/write splitting sends all non-transactional SELECTs to the slaves
 49 --
 50 -- is_in_transaction tracks the state of the transactions
 51 local is_in_transaction       = false
 52
 53 -- if this was a SELECT SQL_CALC_FOUND_ROWS ... stay on the same connections
 54 local is_in_select_calc_found_rows = false
 55

* 初始化变量
* is_in_transaction：false表示转发查询语句到只读连接上
* is_in_select_calc_found_rows：表示查询语句中是否有"SQL_CALC_FOUND_ROWS"

===56-72行===


 56 ---
 57 -- get a connection to a backend
 58 --
 59 -- as long as we don't have enough connections in the pool, create new connections
 60 --
 61 function connect_server()
 62         local is_debug = proxy.global.config.rwsplit.is_debug
 63         -- make sure that we connect to each backend at least ones to
 64         -- keep the connections to the servers alive
 65         --
 66         -- on read_query we can switch the backends again to another backend
 67
 68         if is_debug then
 69                 print()
 70                 print("[connect_server] " .. proxy.connection.client.src.name)
 71         end
 72

* 函数connect_server说明：获取一个到后端的连接
* 初始化is_debug变量

===75-121行===


 73         local rw_ndx = 0
 74
 75         -- init all backends
 76         for i = 1, #proxy.global.backends do
 77                 local s        = proxy.global.backends[i]
 78                 local pool     = s.pool -- we don't have a username yet, try to find a connections which is idling
 79                 local cur_idle = pool.users[""].cur_idle_connections
 80
 81                 pool.min_idle_connections = proxy.global.config.rwsplit.min_idle_connections
 82                 pool.max_idle_connections = proxy.global.config.rwsplit.max_idle_connections
 83
 84                 if is_debug then
 85                         print("  [".. i .."].connected_clients = " .. s.connected_clients)
 86                         print("  [".. i .."].pool.cur_idle     = " .. cur_idle)
 87                         print("  [".. i .."].pool.max_idle     = " .. pool.max_idle_connections)
 88                         print("  [".. i .."].pool.min_idle     = " .. pool.min_idle_connections)
 89                         print("  [".. i .."].type = " .. s.type)
 90                         print("  [".. i .."].state = " .. s.state)
 91                 end
 92
 93                 -- prefer connections to the master
 94                 if s.type == proxy.BACKEND_TYPE_RW and
 95                    s.state ~= proxy.BACKEND_STATE_DOWN and
 96                    cur_idle < pool.min_idle_connections then
 97                         proxy.connection.backend_ndx = i
 98                         break
 99                 elseif s.type == proxy.BACKEND_TYPE_RO and
100                        s.state ~= proxy.BACKEND_STATE_DOWN and
101                        cur_idle < pool.min_idle_connections then
102                         proxy.connection.backend_ndx = i
103                         break
104                 elseif s.type == proxy.BACKEND_TYPE_RW and
105                        s.state ~= proxy.BACKEND_STATE_DOWN and
106                        rw_ndx == 0 then
107                         rw_ndx = i
108                 end
109         end
110
111         if proxy.connection.backend_ndx == 0 then
112                 if is_debug then
113                         print("  [" .. rw_ndx .. "] taking master as default")
114                 end
115                 proxy.connection.backend_ndx = rw_ndx
116         end
117
118         -- pick a random backend
119         --
120         -- we someone have to skip DOWN backends
121

* 创建一个新的或分配一个已有的MySQL Proxy到MySQL Server后端的连接
* 按backends顺序和下列次序选择：
** 优先选取一个可读写的后端，直到该后端空闲连接数达到最小空闲连接数
** 其次选取一个只读的后端，直到该后端空闲连接数达到最小空闲连接数
** 最后选取第一个可读写的后端
* backends顺序是：先rw后ro；命令行配置中出现的先后顺序
* 问题：分配算法和实际测试结果有差异，原因不明。比如min_idle_connections设置是1，但可以建min_idle_connections+1=2个连接
* connected_clients总是0，原因不明，此变量需要脚本自行赋值维护？
* 当客户端连接数小于min_idle_connections+1时，调试信息中cur_idle_connections总显示为0。超过min_idle_connections+1后才显示出数值，不会新建连接，等于min_idle_connections+1

===122-139行===


122         -- ok, did we got a backend ?
123
124         if proxy.connection.server then
125                 if is_debug then
126                         print("  using pooled connection from: " .. proxy.connection.backend_ndx)
127                 end
128
129                 -- stay with it
130                 return proxy.PROXY_IGNORE_RESULT
131         end
132
133         if is_debug then
134                 print("  [" .. proxy.connection.backend_ndx .. "] idle-conns below min-idle")
135         end
136
137         -- open a new connection
138 end
139

* 没看懂。估计是，如果已建立了连接池，则返回PROXY_IGNORE_RESULT，表示忽略接下来服务器发回的handshake，这样就不会建立起新连接
* 函数connect_server结束

===140-164行===


140 ---
141 -- put the successfully authed connection into the connection pool
142 --
143 -- @param auth the context information for the auth
144 --
145 -- auth.packet is the packet
146 function read_auth_result( auth )
147         if is_debug then
148                 print("[read_auth_result] " .. proxy.connection.client.src.name)
149         end
150         if auth.packet:byte() == proxy.MYSQLD_PACKET_OK then
151                 -- auth was fine, disconnect from the server
152                 proxy.connection.backend_ndx = 0
153         elseif auth.packet:byte() == proxy.MYSQLD_PACKET_EOF then
154                 -- we received either a
155                 --
156                 -- * MYSQLD_PACKET_ERR and the auth failed or
157                 -- * MYSQLD_PACKET_EOF which means a OLD PASSWORD (4.0) was sent
158                 print("(read_auth_result) ... not ok yet");
159         elseif auth.packet:byte() == proxy.MYSQLD_PACKET_ERR then
160                 -- auth failed
161         end
162 end
163
164

* 参考
** http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol
* 问题
** [[#read_auth_result没有显示调试信息]]

===165-171行===


165 ---
166 -- read/write splitting
167 function read_query( packet )
168         local is_debug = proxy.global.config.rwsplit.is_debug
169         local cmd      = commands.parse(packet)
170         local c        = proxy.connection.client
171

* read_query函数
* 解析网络包、语句

===172-174行===


172         local r = auto_config.handle(cmd)
173         if r then return r end
174

* 处理PROXY命令(show,set,save,load)，返回PROXY_SEND_RESULT和结果

===175-188行===


175         local tokens
176         local norm_query
177
178         -- looks like we have to forward this statement to a backend
179         if is_debug then
180                 print("[read_query] " .. proxy.connection.client.src.name)
181                 print("  current backend   = " .. proxy.connection.backend_ndx)
182                 print("  client default db = " .. c.default_db)
183                 print("  client username   = " .. c.username)
184                 if cmd.type == proxy.COM_QUERY then
185                         print("  query             = "        .. cmd.query)
186                 end
187         end
188

* 声明本地变量
* 显示调试信息

===189-204行===


189         if cmd.type == proxy.COM_QUIT then
190                 -- don't send COM_QUIT to the backend. We manage the connection
191                 -- in all aspects.
192                 proxy.response = {
193                         type = proxy.MYSQLD_PACKET_OK,
194                 }
195
196                 if is_debug then
197                         print("  (QUIT) current backend   = " .. proxy.connection.backend_ndx)
198                 end
199
200                 return proxy.PROXY_SEND_RESULT
201         end
202
203         proxy.queries:append(1, packet, { resultset_is_needed = true })
204

* 截取QUIT命令，不发送到后端服务器
* 构造查询队列。命令放入查询队列

===205-255行===


205         -- read/write splitting
206         --
207         -- send all non-transactional SELECTs to a slave
208         if not is_in_transaction and
209            cmd.type == proxy.COM_QUERY then
210                 tokens     = tokens or assert(tokenizer.tokenize(cmd.query))
211
212                 local stmt = tokenizer.first_stmt_token(tokens)
213
214                 if stmt.token_name == "TK_SQL_SELECT" then
215                         is_in_select_calc_found_rows = false
216                         local is_insert_id = false
217
218                         for i = 1, #tokens do
219                                 local token = tokens[i]
220                                 -- SQL_CALC_FOUND_ROWS + FOUND_ROWS() have to be executed
221                                 -- on the same connection
222                                 -- print("token: " .. token.token_name)
223                                 -- print("  val: " .. token.text)
224
225                                 if not is_in_select_calc_found_rows and token.token_name == "TK_SQL_SQL_CALC_FOUND_ROWS" then
226                                         is_in_select_calc_found_rows = true
227                                 elseif not is_insert_id and token.token_name == "TK_LITERAL" then
228                                         local utext = token.text:upper()
229
230                                         if utext == "LAST_INSERT_ID" or
231                                            utext == "@@INSERT_ID" then
232                                                 is_insert_id = true
233                                         end
234                                 end
235
236                                 -- we found the two special token, we can't find more
237                                 if is_insert_id and is_in_select_calc_found_rows then
238                                         break
239                                 end
240                         end
241
242                         -- if we ask for the last-insert-id we have to ask it on the original
243                         -- connection
244                         if not is_insert_id then
245                                 local backend_ndx = lb.idle_ro()
246
247                                 if backend_ndx > 0 then
248                                         proxy.connection.backend_ndx = backend_ndx
249                                 end
250                         else
251                                 print("   found a SELECT LAST_INSERT_ID(), staying on the same backend")
252                         end
253                 end
254         end
255

* 没看懂
** 查看查询语句中是否含有SQL_CALC_FOUND_ROWS或LAST_INSERT_ID
** 如果没有LAST_INSERT_ID，则选一个空闲的只读后端，否则后端不变

===256-272行===


256         -- no backend selected yet, pick a master
257         if proxy.connection.backend_ndx == 0 then
258                 -- we don't have a backend right now
259                 --
260                 -- let's pick a master as a good default
261                 --
262                 proxy.connection.backend_ndx = lb.idle_failsafe_rw()
263         end
264
265         -- by now we should have a backend
266         --
267         -- in case the master is down, we have to close the client connections
268         -- otherwise we can go on
269         if proxy.connection.backend_ndx == 0 then
270                 return proxy.PROXY_SEND_QUERY
271         end
272

* 其它非查询语句，选择一个可读写后端
* 如无可选后端，不做后续操作，直接跳出函数

===273-302行===


273         local s = proxy.connection.server
274
275         -- if client and server db don't match, adjust the server-side
276         --
277         -- skip it if we send a INIT_DB anyway
278         if cmd.type ~= proxy.COM_INIT_DB and
279            c.default_db and c.default_db ~= s.default_db then
280                 print("    server default db: " .. s.default_db)
281                 print("    client default db: " .. c.default_db)
282                 print("    syncronizing")
283                 proxy.queries:prepend(2, string.char(proxy.COM_INIT_DB) .. c.default_db, { resultset_is_needed = true })
284         end
285
286         -- send to master
287         if is_debug then
288                 if proxy.connection.backend_ndx > 0 then
289                         local b = proxy.global.backends[proxy.connection.backend_ndx]
290                         print("  sending to backend : " .. b.dst.name);
291                         print("    is_slave         : " .. tostring(b.type == proxy.BACKEND_TYPE_RO));
292                         print("    server default db: " .. s.default_db)
293                         print("    server username  : " .. s.username)
294                 end
295                 print("    in_trans        : " .. tostring(is_in_transaction))
296                 print("    in_calc_found   : " .. tostring(is_in_select_calc_found_rows))
297                 print("    COM_QUERY       : " .. tostring(cmd.type == proxy.COM_QUERY))
298         end
299
300         return proxy.PROXY_SEND_QUERY
301 end
302

* 保证后端数据库和客户端数据库一致
* 显示调试信息

===303-332行===


303 ---
304 -- as long as we are in a transaction keep the connection
305 -- otherwise release it so another client can use it
306 function read_query_result( inj )
307         local is_debug = proxy.global.config.rwsplit.is_debug
308         local res      = assert(inj.resultset)
309         local flags    = res.flags
310
311         if inj.id ~= 1 then
312                 -- ignore the result of the USE 
313                 -- the DB might not exist on the backend, what do do ?
314                 --
315                 if inj.id == 2 then
316                         -- the injected INIT_DB failed as the slave doesn't have this DB
317                         -- or doesn't have permissions to read from it
318                         if res.query_status == proxy.MYSQLD_PACKET_ERR then
319                                 proxy.queries:reset()
320
321                                 proxy.response = {
322                                         type = proxy.MYSQLD_PACKET_ERR,
323                                         errmsg = "can't change DB ".. proxy.connection.client.default_db ..
324                                                 " to on slave " .. proxy.global.backends[proxy.connection.backend_ndx].dst.name
325                                 }
326
327                                 return proxy.PROXY_SEND_RESULT
328                         end
329                 end
330                 return proxy.PROXY_IGNORE_RESULT
331         end
332

* 处理COM_INIT_DB命令的返回结果
* 忽略其它语句的结果

===333-348行===


333         is_in_transaction = flags.in_trans
334         local have_last_insert_id = (res.insert_id and (res.insert_id > 0))
335
336         if not is_in_transaction and
337            not is_in_select_calc_found_rows and
338            not have_last_insert_id then
339                 -- release the backend
340                 proxy.connection.backend_ndx = 0
341         elseif is_debug then
342                 print("(read_query_result) staying on the same backend")
343                 print("    in_trans        : " .. tostring(is_in_transaction))
344                 print("    in_calc_found   : " .. tostring(is_in_select_calc_found_rows))
345                 print("    have_insert_id  : " .. tostring(have_last_insert_id))
346         end
347 end
348

* 参考：http://forge.mysql.com/wiki/MySQL_Internals_ClientServer_Protocol#OK_Packet
* 检查是否在事务内，是否有insert_id。设置backend_ndx=0，下次重新选取后端连接？

===349-364行===


349 ---
350 -- close the connections if we have enough connections in the pool
351 --
352 -- @return nil - close connection
353 --         IGNORE_RESULT - store connection in the pool
354 function disconnect_client()
355         local is_debug = proxy.global.config.rwsplit.is_debug
356         if is_debug then
357                 print("[disconnect_client] " .. proxy.connection.client.src.name)
358         end
359
360         -- make sure we are disconnection from the connection
361         -- to move the connection into the pool
362         proxy.connection.backend_ndx = 0
363 end
364

* 不懂

==问题==

===ERROR 1047 (08S01): Unknown command===
现象：
* 用MySQL Proxy 0.8.1缺省的rw-splitting脚本，客户端连接多次，当每个backend都建立起一定空闲连接（连接数都达到min_idle_connects+1，原因不清）后，再次连接时报错"ERROR 1047 (08S01): Unknown command"。如下：


$ ./mysql -h 127.0.0.1 -P 4040 -u yyyy -pxxxx -e "proxy show config"
+---------+----------------------+-------+---------+
| module  | option               | value | type    |
+---------+----------------------+-------+---------+
| rwsplit | min_idle_connections | 1     | number  |
| rwsplit | is_debug             | true  | boolean |
| rwsplit | max_idle_connections | 4     | number  |
+---------+----------------------+-------+---------+
$ ./mysql -h 127.0.0.1 -P 4040 -u yyyy -pxxxx -e "proxy show config"
+---------+----------------------+-------+---------+
| module  | option               | value | type    |
+---------+----------------------+-------+---------+
| rwsplit | min_idle_connections | 1     | number  |
| rwsplit | is_debug             | true  | boolean |
| rwsplit | max_idle_connections | 4     | number  |
+---------+----------------------+-------+---------+
$ ./mysql -h 127.0.0.1 -P 4040 -u yyyy -pxxxx -e "proxy show config"
ERROR 1047 (08S01): Unknown command

* MySQL数据库版本是5.5的，改成5.1问题消失

原因：
* 好像认证有问题，5.5下read_auth_result了2次。原因待查

If a long-password capable client tries to authenticate to a server that supports long passwords, but the user password provided is actually short, read_auth_result() will be called twice. The first time, auth.packet:byte() will equal 254, indicating that the client should try again using the old password protocol. The second time time read_auth_result()/ is called, auth.packet:byte() will indicate whether the authentication actually succeeded.

解决：
* 无
* 不用5.5版本的数据库

===read_auth_result没有显示调试信息===
现象：
* is_debug设置为true，然而read_auth_result没有显示出debug信息

原因：
* 脚本问题，函数read_auth_result中的变量is_debug没有赋值

解决：
* 在147行之前增加如下代码：


        local is_debug = proxy.global.config.rwsplit.is_debug

* 修改脚本后即可生效，不用重启MySQL Proxy

===proxy set global 报错 ERROR 1064 (42000): You have an error in your SQL syntax===
现象：
* 客户端执行命令"proxy set global xxx.yyy = zzz"，设置MySQL Proxy参数，报语法错误，如下：


mysql> proxy set global rwsplit.min_idle_connections = 2;
ERROR 1064 (42000): You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'proxy set global rwsplit.min_idle_connections = 2' at line 1

原因：
* auto-config.lua脚本问题。不能处理带下划线的参数名称。

解决：
* 修改脚本lib/mysql-proxy/lua/proxy/auto-config.lua
将


local literal = l.R("az", "AZ") ^ 1

改成


local literal = l.R("az", "AZ", "__") ^ 1

* 然后重启MySQL Proxy进程

===proxy set global 报错 ERROR 1105 (07000): (lua) proxy.response.errmsg is nil===
现象：
* 设置is_debug=false报错，如下：


mysql> proxy set global rwsplit.is_debug = false;
ERROR 1105 (07000): (lua) proxy.response.errmsg is nil

原因：
* auto-config.lua脚本问题。当参数值等于false时，parse_value函数返回false，导致"if not val then"判断出错

解决：
* 修改lib/mysql-proxy/lua/proxy/auto-config.lua，handle函数代码
将


if not val then

改为


if val == nil then

* 然后重启MySQL Proxy服务

===proxy save config 保存配置文件大小等于0===
现象：
* proxy save config 命令保存的配置文件，文件大小等于0，要等待一段时间，或重启MySQL Proxy后，其大小才不等于0


-rw-r-----  1 oracle oinstall    0 Jul  4 15:36 "a.log"

原因：
* auto-config.lua脚本问题。文件写操作之后，没有刷新缓冲区

解决：
* 修改lib/mysql-proxy/lua/proxy/auto-config.lua，save函数代码
在


file:write(content)

之后，增加


file:flush(content)

===proxy save config 保存配置文件名带双引号===
现象：
* proxy save config 命令保存的配置文件，文件名包括双引号


-rw-r----- 1 oracle oinstall  186 Jul  4 15:59 "a"

原因：
* auto-config.lua脚本问题。文件名匹配包括双引号

解决：
* 修改lib/mysql-proxy/lua/proxy/auto-config.lua代码
将


local string_quoted  = l.P("\"") * ( 1 - l.P("\"") )^0 * l.P("\"") -- /".*"/

改成


local string_quoted  = l.P("\"") * l.C(( 1 - l.P("\"") )^0) * l.P("\"") -- /".*"/

将


                l.Ct( l.Cc("string") * l.C(string_quoted) +
...
...
         (SAVE / "SAVE" * CONFIG * WS^1 * INTO * l.C(string_quoted)) +
         (LOAD / "LOAD" * CONFIG * WS^1 * FROM * l.C(string_quoted))) * -1)

改成


                l.Ct( l.Cc("string") * string_quoted +
...
...
         (SAVE / "SAVE" * CONFIG * WS^1 * INTO * string_quoted) +
         (LOAD / "LOAD" * CONFIG * WS^1 * FROM * string_quoted)) * -1)

-fin-

ODI FAQ

2011-08-16T00:55:00.003+08:00

去年DW项目实施时写的一些常见问题，主要针对ODI 10g版本，内容较陈旧，仅供参考

===一个变量可以在多个Scenario中使用而不产生冲突吗?===
可以

Recommendations For Working With ODI Variables In Startscen And OdiStartScen Calls ID 423737.1
https://support.oracle.com/CSP/main/article?cmd=show&id=423737.1&type=NOT

If the same Variable is used in multiple simultaneously running Scenarios, it must have the 'Action' parameter set to 'Not Persistent' to avoid Variable value collision from concurrent Scenarios.

===如何处理目录下的所有文件===

ODI Series – Processing all files in a directory: http://john-goodwin.blogspot.com/2009/06/odi-series-processing-all-files-in.html
How to refresh ODI variables from file – Part 1 – Just one value: http://odiexperts.com/?p=243
How to refresh ODI variables from file – Part 2 – Getting all lines, once at time: http://odiexperts.com/?p=273
Getting one or several unknown files from a directory: http://odiexperts.com/?p=1426

===如何删除ODI日志===
How To Avoid Saturating The ODI Log Tables ID 423934.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=423934.1
Session And Scenario Execution Log Tables Used By ODI ID 424663.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=424663.1
Programming ODI To Automatically Purge The Operator Journal ID 423839.1 https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=423839.1
How To Manually Delete A Scenario And Its Related Reports In The ODI Repository With SQL ID 424661.1 https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=424661.1
ODI snippets: Purge Log and shrink space: http://www.business-intelligence-quotient.com/?p=539

===ODI User Function怎么用?===
ODI User Functions: A Case Study: http://blogs.oracle.com/dataintegration/2009/09/odi_user_functions_a_case_stud.html
http://john-goodwin.blogspot.com/2009/06/odi-series-quick-look-at-user-functions.html

===如何并行执行过程===
Parallel Processing in ODI: http://blogs.oracle.com/dataintegration/2009/11/parallel_processing_in_odi.html
How To Run ODI Processes In Parallel And Using OdiWaitForChildSession ID 443858.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=443858.1
Asynchronous -Parallel Execution: http://odiexperts.com/?p=1180

===ODI的学习资源有哪些===
Oracle Data Integrator: http://www.oracle.com/technology/products/oracle-data-integrator/index.html
Oracle Data Integrator Release 10.1.3.6 Documentation Library: http://download.oracle.com/docs/cd/E15985_01/index.htm
Oracle Data Integrator Downloads: http://www.oracle.com/technology/software/products/odi/index.html
Oracle Data Integrator应用指南: http://www.docin.com/p-32319219.html
Oracle Data Integrator/Sunopsis, Releases and Patches ID 456313.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=456313.1
My Oracle Support searching keyword odi: http://support.oracle.com/
Data Integration Forum: http://forums.oracle.com/forums/forum.jspa?forumID=374
Developing a Knowledge Module in Oracle Data Integrator: http://www.oracle.com/technology/pub/articles/bethke-odi.html
ODI Resources for Beginners: Getting Started with Oracle Data Integrator: http://www.business-intelligence-quotient.com/?p=379
Oracle Data Integration Resource Kit. Included is the State of the Data Integration Market White Paper, in addition to additional key Data Integration resources.: http://www.oracle.com/webapps/dialogue/dlgpage.jsp
Oracle by Example Series: Oracle Data Integrator: http://www.oracle.com/technology/obe/fusion_middleware/odi/index.html
http://www.oracle.com/technology/obe/fusion_middleware/ODI/index.html
Data Integration and Management: http://blogs.oracle.com/dataintegration/
ODI EXPERTS: http://odiexperts.com/
More to life than this...: http://john-goodwin.blogspot.com/
BI-Quotient: http://www.business-intelligence-quotient.com/
Rittman Mead: http://www.rittmanmead.com
Business Intelligence – Oracle: http://oraclebizint.wordpress.com/category/oracle-data-integrator/

Oracle Czech BI/DW Blog: http://bidwcz.blogspot.com/
Technology & Other Worldly Issues: http://askankit.blogspot.com/

https://dbinfo.no.sohu.com:4443/apex/p?n=8247611144080585
https://dbinfo.no.sohu.com:4443/apex/p?n=8247701710096829


Note: The repository structure changes slightly between major versions and in particular from 11g onwards.
In this case, it is recommended to use the SDK instead of querying the repository tables directly.

===如何对数据源UNION操作===
Is It Possible To Use A UNION Operation Of Two Datastores As A Source For An ODI Integration Interface? ID 423731.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=423731.1
KM FOR UNION BASED ON IKM SQL CNTRL APPEND: http://odiexperts.com/?p=1164

===ODI授权===
Q:

开发想修改md_xxx中dm_agent，想定义为缓慢变化维，我用security manager通过profile给他授予了Object，Model Folder,Model,Sub-Model,Datastore 的所有权限，然后把md_xxx拖到了该用户下instances目录下，指定repository为wk_dev。增加这些权限后，开发说还是报错：com.sunopsis.core.SecurityAccessException: You are not authorized to Edit the Object:SNP_JOIN : DM_AGENT_FK_CHANNEL / 6001
难道我还得把相应的datastore也拖到用户下? ODI授个权怎么就这么麻烦呢? 这种操作到底需要授予哪些权限?

A:

we don't know

===如何用动态文件名===
Using A Dynamically Specified File Name Or Directory Name With ODI ID 423635.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=423635.1
Using Parameters in ODI: The Dynamic File Name Example: http://blogs.oracle.com/dataintegration/2009/04/using_parameters_in_odi_the_dy_1.html

===Operator中如何显示出变量值?===
Find the Value of the Variable in Operator ?: http://odiexperts.com/?p=855
ODI Variables and the Operator Module: http://www.business-intelligence-quotient.com/?p=762


One Comment on “ODI Variables and the Operator Module”

  1.
    #1 Craig Stewart said at 3:18 pm on March 17th, 2010:

    Uli
    Another great article – another snippet to follow-up. If the variable has been passed into the execution as a parameter, then if you use odiRef.getSession(”SESS_PARAMS” ) it will printout the values of the parameters. To make this readable in the execution, I usually put in a procedure with a Java Beanshell step, with the code:
    /* */
    As it is in /* */ comments, it is ignored, and does not look like an error/kludge, but prints out neatly in the log.
    Shame there is no similar thing for the variables set dynamically!
    Craig

Using Jython Scripting Features In ODI FAQ ID 566475.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=566475.1#aref78


10. Displaying ODI Variables Values In ODI Operator Log

Is there any technique whereby it is possible, in the ODI Operator Log, to display the values of ODI Variables which have been initialized from runtime parameters when launching Scenarios?
Such a technique would allow the recording of Variable values during runtime and would be useful for execution audit operations.

Two alternative solutions are possible:

In the case of OdiStartScen (SnpsStartScen) tool, it is possible to specify the name of the Session with the "-SESSION_NAME" parameter. In this case, pass the value of the ODI Variable to this parameter for display in ODI Journal.

A Jython script step such as the following, in a Knowledge Module, will allow the display of the ODI Variable value in the Execution tab of the step :
a = 'Table = '+ '#YOURVARIABLE'
raise(a)


Such a Jython script will require the "Ignore errors" checkbox to be checked in the Knowledge Module step. Do not forget that such an Integration Interface must be called from an ODI Package including steps which Declare and Refresh the ODI Variable.

Variable values cannot be displayed in this case because the Agent records data in ODI Operator Log before the ODI process that substitutes Variables by their values, and before sending the values to the technology be be directly used or bound.

===如何重新生成scenario===
ODI: Automating deployment of scenarios to production in Oracle Data Integrator: http://www.business-intelligence-quotient.com/?p=340
What Will Each Generation Mode (Replace, Re-generate, Creation) Of ODI Project "Generate All Scenarios" Option Do? ID 555709.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=555709.1
Generate Multiple Scenarios: http://odiexperts.com/?p=456
In ODI What Is The Difference Between Generation And Regeneration Of A Scenario ID 733103.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=733103.1

How To Ensure That The Regeneration Of An ODI Scenario Is Successful? ID 804583.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=804583.1

===生成场景报错ORA-12899===
Q:

生成场景时报如下错误
java.sql.SQLException: ORA-12899: value too large for column "W_DEV"."SNP_SCEN_STEP"."VAR_VALUE" (actual: 265, maximum: 250)
如何解决？

A:

变量赋值不能超过250字符。可以将一个大的变量分成多个小的变量，最后再拼到一起。

===用OdiFileWait报错===
Q:

程序中使用OdiFileWait检查文件是否存在，文件可以被ls看到，目录权限drwxr-xr-x，文件权限-rw-r--r--，但是OdiFileWait总报错说No File Found.

A:

# TIMEOUT时间太短，应该设置长一些，比如10000ms
# 文件名PATTERN中不能带目录名，比如#v_dirname/#v_filename，只能是纯文件名
# 改用别的方式检查文件是否存在，比如OdiOSCommand, sh -c "test -f #v_dirname/#v_filename"

===SQLLDR导数据的字段顺序出错===
Q:

用LKM File to Oracle (SQLLDR)导数据，报错字段格式不对。检查日志和生成的控制文件，发现控制文件中字段的顺序变了，跟文件中不一致

A:

因为在Interface配置的Target Datastore有些字段execute on source,有些execute on staging。KM生成的控制文件中，字段顺序自动发生了变化，execute on source的字段在前，execute on staging的在后。所以相关字段统一使用execute on source或staging...，顺序才不会变。

===IKM插入数据报错 ORA-01031: insufficient privileges===
Q:

IKM SQL Control Append在Insert new rows这步报错


1031 : 42000 : java.sql.SQLException: ORA-01031: insufficient privileges

java.sql.SQLException: ORA-01031: insufficient privileges

执行的语句是


insert /*+ append */ into  ods.ODS_ORACLE_TABXXX
(
….
from  ODI_WORK.I$_ODS_ORACLE_TABXXX

直接用sqlplus以odi work用户在ods数据库上执行同样语句，报错


    ,ods.seq_ods_oracle_tabxxx_id.nextval,
          *
ERROR at line 88:
ORA-01031: insufficient privileges

A:

ods库上的odi work用户没有相应权限，需要手工授予


grant select any sequence to ODI_WORK;

===如何实现反连接?===
Q:

ODI中如何实现not in, not exists
比如:


select * from t1 where not exists (select null from t2 where t2.c1=t1.c1)

A:

How To Set Up Negative Join Operations In An ODI Integration Interface ? ID 742144.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=742144.1
Using Oracle Outer Join Operations Such As LIKE, IN, NOT IN, IS NULL ...In ODI Integration Interfaces. ID 424034.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&id=424034.1

比如


select * from t1 where not exists (select null from t2 where t2.c1=t1.c1)

用left outer join实现:


Interface->Diagram->Sources

* t1 left outer join t2
** 注意不能选中left outer join复选框, 必须手工写(+)号
**Implementation填t1.c1=t2.c1(+), 设置为execute on staging
* t2增加filter, Implementation填t2.c1 is null, 设置为execute on staging

===刷新变量出错No value to affect to this variable for DefDate===
Q:

Package中刷新变量，报错Error: No value to affect to this variable for DefDate
意思是说select刷新语句没有返回结果。
然而数据库中运行select语句可以查到记录，有返回结果。

Package是按照Best practice，先声明变量，再刷新的，使用上应该没有错。

刷新语句是:


select to_char(t1_id) from admin_odi.t1
where name='#v_t1_name'

* 将where子句改成where name='p_city_day'，也报同样错。
* 如果注释掉where子句，改成select to_char(p1_id) from admin_odi.t1，则刷新成功，取得第一条记录的结果。

A:

* 原因不明。
* 删掉Diagram中变量声明和刷新的步骤，重新定义这2步，即可。

===调用场景参数中带双引号的转义===
Q:

用如下命令调用一个场景失败，报Unmatching quotes错误


OdiStartScen "-SCEN_NAME=T1_INVOKER" "-SCEN_VERSION=001" "-TEST_XIEWEN.v_t1_name=p_city_day" "-TEST_XIEWEN.v_scen_name=P_CITY_DAY" "-TEST_XIEWEN.v_scen_params=\"-ODS_FROM_ONLINE_CITY.v_start_date=#TEST_XIEWEN.v_start_date\" \"-ODS_FROM_ONLINE_CITY.v_end_date=#TEST_XIEWEN.v_end_date\" \"-ODS_FROM_ONLINE_CITY.v_date_format=#TEST_XIEWEN.v_date_format\""

应如何转义双引号？

A:

现在用的方法是，先用一段特殊字符代表双引号，在被调用的场景中再将特殊字符替换回双引号

调用场景时，特殊字符\\u0022代表双引号:


OdiStartScen "-SCEN_NAME=T1_INVOKER" "-SCEN_VERSION=001" "-TEST_XIEWEN.v_t1_name=p_city_day" "-TEST_XIEWEN.v_scen_name=P_CITY_DAY" "-TEST_XIEWEN.v_scen_params=\\u0022-ODS_FROM_ONLINE_CITY.v_start_date=#TEST_XIEWEN.v_start_date\\u0022 \\u0022-ODS_FROM_ONLINE_CITY.v_end_date=#TEST_XIEWEN.v_end_date\\u0022 \\u0022-ODS_FROM_ONLINE_CITY.v_date_format=#TEST_XIEWEN.v_h1\\u0022"

场景中再替换回双引号：


select replace(q'{#v_scen_params}','\\u0022','"') from dual

===改resource name后interface取的还是旧表名===
Q:

为了测试用，将一个表data store的resource name改成另一个临时建的新表名，点refresh number of rows显示行数也和新表名相符, 然而运行interface，取到的还是旧表数据。为什么？

A:

* LKM Oracle to Oracle (DBLINK)_corp使用了odiRef.getSrcTablesList("", "[SCHEMA].[TABLE_NAME]获得源表表名
** TABLE_NAME表示逻辑表名,RES_NAME才表示物理表名
** TABLE_NAME Logical name of the source datastore
** RES_NAME Physical access name of the resource.......
* 所以修改resource name不管用，必须修改data store的名字。

===Historize变量没取到最新值===
Q:

有一个Historize变量v_etl_sysdate每天都刷新为当天的日期，通过变量的历史可以看到。

package查看这个变量，得到的不是最后一次刷新的值(2010-07-16)，是历史记录中的第一个值(2010-04-09)。


SQL> select * from SNP_VAR_DATA where var_name='S2ODS.v_etl_sysdate' order by FIRST_DATE;

VAR_NAME
------------------------------------------------------------------------------------------------------------------------------------------------------
CONTEXT_CODE                                          I_VAL VAR_D
---------------------------------------------------------------------- ---------- -------------------
VAR_V
------------------------------------------------------------------------------------------------------------------------------------------------------
  VAR_N FIRST_DATE      FIRST_I_USER LAST_DATE      LAST_I_USER I_TXT_VAR_T
---------- ------------------- ------------ ------------------- ----------- -----------
S2ODS.v_etl_sysdate
EMPDATAB                                              6591
2010-04-09
      2010-04-09 00:29:59        2010-04-09 00:29:59

S2ODS.v_etl_sysdate
EMPDATAB                                            21382
2010-04-10
      2010-04-10 00:29:59        2010-04-10 00:29:59

...
...
S2ODS.v_etl_sysdate
EMPDATAB                                            1141971
2010-07-15
      2010-07-15 01:00:00        2010-07-15 01:00:00

S2ODS.v_etl_sysdate
EMPDATAB                                            1142420
2010-07-16
      2010-07-16 01:00:00        2010-07-16 01:00:00


98 rows selected.

SQL>

A:

* 直接执行package能取到最新值。
* 执行scenario, 如果生成scenario时，选择了v_etl_sysdate做参数，则以后运行scenario该参数默认值都是历史第一个值。
* 生成scenario不选择该变量做参数，执行也可得到最新值。

===ODI 11gR1===
document: http://download.oracle.com/docs/cd/E14571_01/odi.htm
download: http://edelivery.oracle.com/EPD/Download/get_form?egroup_aru_number=11571971

===如有error/discard记录，LKM SQLLDR导入失败===
Q:

如果有error或discard记录，LKM SQLLDR导入失败

A:

sqlloader not load file in oracle when get invalid rows.: http://forums.oracle.com/forums/thread.jspa?threadID=1043579


This is a bug 8560194 with ODI LKM File to Oracle (SQLLDR) and it has not been resolved yet.
SQLLDR if successful returns 0
SQLLDR if unsuccessful returns 1
SQLLDR if successful but even 1 record erred out to .bad or .dsc file returns 2

So, for ODI anything that is not 0 is an error.

To resolve this bug, you will have to customize this KM and change the step "Invoke SQLLDR" to selectively handle each error condition.


Duplicate the KM.
Goto the step Call sqlldr. and change the technology to Jython.

Replace the existing code with the following:

import os
retVal = os.system(r'sqlldr control=<%=snpRef.getSrcTablesList("", "[WORK_SCHEMA]/[TABLE_NAME].ctl", "", "")%> log=<%=snpRef.getSrcTablesList("", "[WORK_SCHEMA]/[TABLE_NAME].log", "", "")%> userid=<%=snpRef.getInfo("DEST_USER_NAME")%>/<%=snpRef.getInfo("DEST_PASS")%>@<%=snpRef.getInfo("DEST_DSERV_NAME")%> > <%=snpRef.getSrcTablesList("", "[WORK_SCHEMA]/[TABLE_NAME].out", "", "")%>')

if retVal == 1 or retVal > 2:
  raise 'SQLLDR failed. Please check the <%=snpRef.getSrcTablesList("", "[WORK_SCHEMA]/[TABLE_NAME].log", "", "")%> for details '


Hope that helps

===显示Session运行状态有问题===
Q:

Operator日志显示某Session总是Running状态，在后台查询已无相应数据库会话

A:

该会话可能是从本机No agent发起的，当会话运行中，退出了客户端(Designer,Operator)。客户端退出，会话也随之终止，但会话状态不会更新到ODI系统表。

===过程中使用绑定变量，报错Missing parameter===
Q:

Procedure中一条语句使用到了bind variable:


begin
...
UPDATE <%=odiRef.getOption("TABLE_NAME")%>
  SET STATUS=<%=odiRef.getOption("T1_STATUS")%>
  WHERE T1_ID=<%=odiRef.getOption("T1_ID")%>;
...
end;

P1_ID赋值为":ADMIN_ODI.v_t1_ID"，开头的冒号表示使用绑定变量

执行时出错:


com.sunopsis.sql.SnpsMissingParametersException: Missing parameter
  at com.sunopsis.sql.SnpsQuery.completeHostVariable(SnpsQuery.java)
  at com.sunopsis.sql.SnpsQuery.updateExecStatement(SnpsQuery.java)
...

日志显示，最后生成的语句是:


begin
...
UPDATE #ADMIN_ODI.v_admin_odi_t1
  SET STATUS=2
  WHERE T1_ID=:ADMIN_ODI.v_t1_ID;
...
end;

A:

ODI把UPDATE语句最后的分号当成变量名的一部分，所以找不到变量，就报错了。

分号换行写，或前面加空格与变量名分开，即可。

===OdiOSCommand双连字符问题===
Q:

* OdiOSCommand调用命令sh -c "rsync -q --dry-run ...."
* 从Operator操作日志观看生成的命令，double hyphen及其后字符都没有了
* 将双连字符改成\u002d\u002d，生成命令没错，执行报错"命令返回:2"
* 检查出错原因，好像还是因为双连字符后被截断了，导致命令不完整:


sh: -c: line 0: unexpected EOF while looking for matching `"'
sh: -c: line 1: syntax error: unexpected end of file

A:

OdiOutFile先将执行的命令写到一个脚本内，然后OdiOSCommand调用此脚本

===SNP_SESSION.SESS_STATUS字段代表什么含义?===
* D = Done
* W = Waiting
* E = Error
* R = Running
* M = Warning
* Q = Queued

===SQLLDR LKM如何实现只选择几列导入===
Q:

* 一个文件有A,B,C 3列，数据库表中有B,C 2列
* interface mapping只映射了B,C列, km生成的CTL文件内也只有这2列, 于是字段不匹配，sqlldr导入失败

A:

对所有没有mapping的字段，建filter, 填字段名 is not null or 字段名 is null

===Scheduler Agent和Work Repository是一一对应的吗===
Q:

odiparam.sh有个参数ODI_SECU_WORK_REP，表示agent连接哪个工作资料库

10g中，Scheduler Agent在启动时连接该工作资料库，获取scheduler定时任务

11g，情况好像有所不同，启动agent后，日志显示:


2010-08-24 20:08:38.177 NOTIFICATION ODI-1128 Agent pa_emods_dev is starting. Application Server: STANDALONE. Agent Version: 11.1.1.3.0 - 23/06/2010
. Port: 21911. JMX Port: 22911
2010-08-24 20:09:01.667 NOTIFICATION ODI-1136 Starting Schedulers on Agent pa_ods_dev
2010-08-24 20:09:01.853 NOTIFICATION ODI-1111 Agent pa_emods_dev started. Agent version: 11.1.1.3.0 - 23/06/2010. Port: 21911. JMX Port: 22911.
2010-08-24 20:09:01.907 NOTIFICATION ODI-1137 Scheduler started for work repository REPWPROD on Agent pa_ods_dev
2010-08-24 20:09:02.012 NOTIFICATION ODI-1137 Scheduler started for work repository REPWDEV on Agent pa_ods_dev

貌似连接了所有工作资料库，一个agent就管理了所有scheduler任务

odiparam.sh对该变量的注释也有所变化：


# The following work repository name variable is required by startcmd (when the specified
# command needs to connects to the work repository), startscen, and restartsession scripts.
# The agent startup and agentstop programs do not need this variable.
#
#  ODI_SECU_WORK_REP
#    Name of the Work Repository to connect. This Work Repository must
#    be attached to the master repository.

A:

测试了下，好像是这样
* 10g scheduler agent从odiparam.sh指定的work repository取出context和logical agent等于当前physical agent的任务
* 11g agent从所有的work repositories取出context和logical agent等于当前physical agent的任务

7.2.2 Agent Startup and Shutdown Cycle: http://download.oracle.com/docs/cd/E14571_01/core.1111/e10106/odi.htm#ASHIA4972


When the Oracle Data Integrator agent starts, it first reads its configuration, which includes master repository connection information. Then the agent connects to each of the work repositories attached to this master repository and removes stale sessions. Stale sessions are sessions that are incorrectly indicated in the work repository as running on this given agent. Stale sessions may result from an agent being stopped without being able to stop these sessions gracefully. As the agent restarts, it identifies the stale sessions and moves them to an error state.

From that point, the agent can retrieve and compute the schedules available for it in each work repository. Once this phase is complete, the agent waits for incoming sessions requests to start their processing, and is also able to start sessions based on its schedules.

===如何使用子查询===
Using subqueries in Oracle Data Integrator (ODI) interfaces for complex data integration requirements: http://www.business-intelligence-quotient.com/?p=621
Subselect, derived tables, and subqueries in ODI 11G: http://www.business-intelligence-quotient.com/?p=1045

===生成场景报错An object will be in an unidentified state===
Q:

https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&doctype=FAQ&id=471564.1#aref17


2. "An object will be in an unidentified state...." Message Is Randomly Displayed
The following message is displayed randomly:
"An object will be in an unidentified state (it is in the state Not found in the database and Update is currently being performed on it) " for example, when refreshing an Oracle Data Integrator Variable, running a Scenario...

Such a problem has been attributed to the ODI Log related tablespaces being full.

Workarounds are the following:

Increase the size of the appropriate tablespaces containing the ODI Work Repositories.
Purge the Log tables in ODI Log (Operator).

A:

* 删除日志

* 关了重新打开，有时也能好

===MySQL建temporary indexes的问题===
Q:

LKM建临时索引使用如下语句:


Create <%=odiRef.getTemporaryIndex ("INDEX_TYPE_CLAUSE") %> Index <%=odiRef.getObjectName(odiRef.getTemporaryIndex ("IDX_NAME"),"W") %> on <%=odiRef.getTemporaryIndex ("COLL_NAME")%>
<%=odiRef.getTemporaryIndexColList( "(", " [CX_COL_NAME] ", ", ", ")" )%>

生成的SQL语句是:


create ... index aaa.ix$_bbbb on .....;

索引名之前带上了数据库名

MySQL不支持这样写，应去掉数据库名:aaa.

所以将KM写为


Create <%=odiRef.getTemporaryIndex ("INDEX_TYPE_CLAUSE") %> Index ", "W").substring(odiRef.getCatalogName("W").length()+1)?> on <%=odiRef.getTemporaryIndex ("COLL_NAME")%>
<%=odiRef.getTemporaryIndexColList( "(", " [CX_COL_NAME] ", ", ", ")" )%>

A:

good

===OdiSqlUnload 报错内存不够===
Q:

使用OdiSqlUnload导出数据，大约28万条记录，报OutOfMemoryError


java.lang.OutOfMemoryError: Java heap space
  at com.mysql.jdbc.MysqlIO.nextRowFast(MysqlIO.java:1585)
  at com.mysql.jdbc.MysqlIO.nextRow(MysqlIO.java:1409)
  at com.mysql.jdbc.MysqlIO.readSingleRowSet(MysqlIO.java:2886)
  at com.mysql.jdbc.MysqlIO.getResultSet(MysqlIO.java:476)
  at com.mysql.jdbc.MysqlIO.readResultsForQueryOrUpdate(MysqlIO.java:2581)
  at com.mysql.jdbc.MysqlIO.readAllResults(MysqlIO.java:1757)
  at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2171)
  at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2568)
  at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2113)
  at com.mysql.jdbc.PreparedStatement.executeQuery(PreparedStatement.java:2275)
  at com.sunopsis.sql.SnpsQuery.executeQuery(SnpsQuery.java:602)
  at com.sunopsis.dwg.tools.SqlUnload.actionExecute(SqlUnload.java:327)
  at com.sunopsis.dwg.function.SnpsFunctionBase.execute(SnpsFunctionBase.java:273)
  at com.sunopsis.dwg.dbobj.SnpSessTaskSql.execIntegratedFunction(SnpSessTaskSql.java:3185)
  at com.sunopsis.dwg.dbobj.SnpSessTaskSql.execIntegratedFunction(SnpSessTaskSql.java:1414)
  at com.sunopsis.dwg.dbobj.SnpSessTaskSql.treatTaskTrt(SnpSessTaskSql.java:2785)
  at com.sunopsis.dwg.dbobj.SnpSessTaskSql.treatTask(SnpSessTaskSql.java:2515)
  at com.sunopsis.dwg.dbobj.SnpSessStep.treatAttachedTasks(SnpSessStep.java:534)
  at com.sunopsis.dwg.dbobj.SnpSessStep.treatSessStep(SnpSessStep.java:449)
  at com.sunopsis.dwg.dbobj.SnpSession.treatSession(SnpSession.java:1954)
  at com.sunopsis.dwg.dbobj.SnpSession.treatSession(SnpSession.java:1818)
  at oracle.odi.runtime.agent.processor.impl.StartScenRequestProcessor$2.doAction(StartScenRequestProcessor.java:559)
  at oracle.odi.core.persistence.dwgobject.DwgObjectTemplate.execute(DwgObjectTemplate.java:224)
  at oracle.odi.runtime.agent.processor.impl.StartScenRequestProcessor.doProcessStartScenTask(StartScenRequestProcessor.java:481)
  at oracle.odi.runtime.agent.processor.impl.StartScenRequestProcessor$StartScenTask.doExecute(StartScenRequestProcessor.java:1040)
  at oracle.odi.runtime.agent.processor.task.AgentTask.execute(AgentTask.java:114)
  at oracle.odi.runtime.agent.support.DefaultAgentTaskExecutor$2.run(DefaultAgentTaskExecutor.java:82)
  at java.lang.Thread.run(Thread.java:619)

A:

增加odi agent java虚拟机内存

"java heap OutOfMemory" Error When Loading A Large Amount Of Data From Mysql With ODI ID 737309.1: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&doctype=PROBLEM&id=737309.1


This issue is due to a MySQL JDBC Driver issue.

When selecting a large amount of data, the MySQL JDBC Driver loads the entire ResultSet returned by the select to the JVM's memory.

This seems to be the case for any version of MySQL JDBC Driver.

* http://dev.mysql.com/doc/refman/5.1/en/connector-j-reference-implementation-notes.html


ResultSet

By default, ResultSets are completely retrieved and stored in memory. In most cases this is the most efficient way to operate, and due to the design of the MySQL network protocol is easier to implement. If you are working with ResultSets that have a large number of rows or large values, and can not allocate heap space in your JVM for the memory required, you can tell the driver to stream the results back one row at a time.

To enable this functionality, you need to create a Statement instance in the following manner:

stmt = conn.createStatement(java.sql.ResultSet.TYPE_FORWARD_ONLY,
        java.sql.ResultSet.CONCUR_READ_ONLY);
stmt.setFetchSize(Integer.MIN_VALUE);

The combination of a forward-only, read-only result set, with a fetch size of Integer.MIN_VALUE serves as a signal to the driver to stream result sets row-by-row. After this any result sets created with the statement will be retrieved row-by-row.

* 经试验,odiunload 参数fetchsize改小，比如改成10，能成功

===索引===
Q:

建索引

A:


create index w_dev.i_SNP_SESSION_psn on w_dev.SNP_SESSION(PARENT_SESS_NO) online;
create index w_prod.i_SNP_SESSION_psn on w_prod.SNP_SESSION(PARENT_SESS_NO) online;
...


create index w_dev.i_SNP_OBJ_STATE_IO on w_dev.SNP_OBJ_STATE(I_INSTANCE,I_OBJECT) online;
create index w_prod.i_SNP_OBJ_STATE_IO on w_prod.SNP_OBJ_STATE(I_INSTANCE,I_OBJECT) online;
...


create index w_prod.i_SNP_SESSION_an_ss on w_prod.SNP_SESSION(AGENT_NAME,SESS_STATUS) online;
create index w_dev.i_SNP_SESSION_an_ss on w_dev.SNP_SESSION(AGENT_NAME,SESS_STATUS) online;

===导入报错 ORA-00001: AK_SNP_GRP_STATE===
Q:

导入一个项目时报错：


java.sql.SQLException: ORA-00001: unique constraint (xxxx.AK_SNP_GRP_STATE) was happend
...

改用utf-8格式导出导入，导入报错


... integrity constraint (xxxxx.FK_TXT) violated - Parent Key are not found.
...

A:

因有人以中文系统创建了项目，项目中带有中文字符，如Markers是中文名称，如：优先级、进度、笑脸

将中文标识改为英文后，导出导入成功

===ODI Sendmail tool 发中文显示乱码===
Q:

用ODI sendmail tool发中文邮件，显示为乱码

A:

标题和内容编码不对，默认用了操作系统的语言设置

方法1：ODI Agent环境变量LANG设置GBK，LANG=zh_CN.gbk，然后重启agent

方法2：自己编写jython程序，支持编码，参考Unprintable And UTF-8 Characters When Sending Emails With The ODI SendMail Tool ?: https://support.oracle.com/CSP/main/article?cmd=show&type=NOT&doctype=PROBLEM&id=423953.1。未能实现

===ODI Agent不断重启===
Q:

按问题[#ODI Sendmail tool 发中文显示乱码]所述，设置LANG=zh_CN.gbk后，odi agent反复重启

A:

经检查，是/etc/init.d/odiagentctl condstart重启了进程

LANG改成zh_CN后，ping日志内容变成中文，脚本无法识别中文，因此误判，导致重启

只要将


grep -q -- "Invalid parameter:-SECURITY_DRIVER"

改成


grep -q -- ":-SECURITY_DRIVER"

即可

--fin--

Distributed read consistency 分布式读一致性

2011-02-22T23:24:00.004+08:00

Distributed read consistency 分布式读一致性

据文档Managing Read Consistency称，在分布式环境中，由于SCN不同步，会导致读出旧数据。比如：远端数据库更新了一个表的记录，并提交。然后，本地数据库通过数据库链接查询该表，得到的是更新前的数据，再次查询才得到更新后的数据。

多次试验后均未能重现文档所描述的现象，总是能读出最新提交的数据，原因不明。

外部链接：
Distributed Query Returns Different Results on Consecutive Runs when no Changes Exist in the Base Tables. [ID 561471.1]
Bug 6933259: DISTRIBUTED QUERY RETURNS WRONG RESULTS
Bug 611416: SELECT AFTER UPDATE AND COMMIT DOESN'T SHOW UPDATED COLUMNS

-fin-

Using dNFS clonedb for testing - 使用dNFS搭建测试数据库

2011-02-21T00:52:00.005+08:00

Using dNFS clonedb for testing
使用dNFS搭建测试数据库

1. 配置dNFS

1.1. 挂载NFS
编辑/etc/fstab，如：

192.168.1.1:/dbak/xxods /home/oracle/xxodsnew    nfs     rw,bg,hard,intr,nolock,tcp,rsize=32768,wsize=32768,nocto,retry=480  0       0

1.2. 启用nfsodm库
将原odm库改名，并建为nfsodm库的符号连接，如：

ln -s libnfsodm11.so libodm11.so

重启数据库，告警日志显示"Oracle instance running with ODM"，说明dNFS已启用

...
  pga_aggregate_target     = 1G
  dg_broker_start          = FALSE
  diagnostic_dest          = "/opt/oracle"
Oracle instance running with ODM: Oracle Direct NFS ODM Library Version 3.0
Fri Dec 31 18:13:43 2010
...

1.3. 设置oradism可执行程序的属性
设置oradism可执行程序的属性setuid，执行

root.sh -silent

oradism可执行程序的属性变为：

-rwsr-x--- 1 root   oinstall     68432 Sep  4 22:21 oradism

如不设置，告警日志中将可能会报错"Direct NFS: please check that oradism is setuid"，如：

...
space available in the underlying filesystem or ASM diskgroup.
Fri Dec 31 18:19:19 2010
alter database backup controlfile to '/home/oracle/xxodsnew/control_bak.ctl'
Direct NFS: please check that oradism is setuid
Fri Dec 31 18:20:19 2010
Direct NFS: please check that oradism is setuid
Fri Dec 31 18:21:19 2010
Direct NFS: please check that oradism is setuid
Fri Dec 31 18:22:19 2010
Direct NFS: please check that oradism is setuid
Fri Dec 31 18:23:20 2010
Completed: alter database backup controlfile to '/home/oracle/xxodsnew/control_bak.ctl'
Fri Dec 31 18:23:51 2010
Starting background process SMCO
Fri Dec 31 18:23:51 2010
SMCO started with pid=28, OS id=18071
...

1.4. 检查视图
v$dnfs_servers: dnfs服务器
v$dnfs_channels: dnfs网络路径
v$dnfs_files: dnfs文件
v$dnfs_stats: dnfs状态

如：

SQL> SQL> select * from v$dnfs_servers;

        ID
----------
SVRNAME
--------------------------------------------------------------------------------
DIRNAME
--------------------------------------------------------------------------------
   MNTPORT    NFSPORT      WTMAX      RTMAX
---------- ---------- ---------- ----------
         1
192.168.1.1
/dbak/xxods
       721       2049          0          0

外部链接：
Direct NFS: FAQ [ID 954425.1]
Step by Step - Configure Direct NFS Client (DNFS) on Linux [ID 762374.1]
3.2.3 Deciding to Use Direct NFS for Data Files

Mount Options for Oracle files when used with NAS devices [ID 359515.1]

2. 在dnfs上搭建测试库

使用了dnfs写时拷贝技术（copy-on-write），测试库只写入修改的数据块，从而减少了其占用空间。

主要步骤如下：
2.1. 生产机导出nfs目录，试机挂载nfs目录
2.2. 创建生产库数据文件备份，用rman image复制，操作系统复制，或存储快照。数据文件备份存放在nfs目录上。
2.2. 生产库控制文件建跟踪备份
2.4. 测试库修改控制文件，将数据文件路径改为nfs备份目录。修改参数文件一些相应参数
2.5. nomount方式启动测试库，建控制文件
2.6. 执行dbms_dnfs.clonedb_renamefile存储过程，将数据文件备份改名到测试目录下。实际上没有真的改名，备份是只读的，不会有任何改动
2.8. 恢复并打开测试数据库

外部链接：
Oracle Database 11g Direct NFS Clonedb Feature – Part I.
Oracle Database 11g Direct NFS Clonedb Feature – Part I (and a half).
Direct NFS (DNFS) Clonedb in Oracle Database 11g Release 2 (Patchset 11.2.0.2)
Clone your dNFS Production Database for Testing [ID 1210656.1]

NOTE 1210656.1 from MOS:

Clone your dNFS Production Database for Testing [ID 1210656.1]

--------------------------------------------------------------------------------

Modified 17-FEB-2011 Type ANNOUNCEMENT Status PUBLISHED

In this Document
What is being announced?
What do you need to do?

--------------------------------------------------------------------------------

Applies to:
Oracle Server - Enterprise Edition - Version: 11.1.0.6 to 11.2.0.1 - Release: 11.1 to 11.2
Information in this document applies to any platform.
Oracle RDBMS
dNFS
What is being announced?
Cloning a production database is an often needed procedure in customer environments to develop and test new application patches. When new OS releases, Storage software, Application version has to be installed in a production environment, it calls for a thorough testing using the production data. Today this is achieved by making a copy of the production datafiles in a test environment. In addition to the test environment, copies of the production database also made in development environment where application developers are creating their new applications and testing them. All of these require huge amount of storage space to be allocated and managed.

Database cloning process involves copying the entire production database files into the new environment where the development or testing takes place. In addition when customers have multiple test environments, the files need to be copied into each of these test environments. dNFS provides a new feature called clonedb which allows test databases to be cloned instantaneously. dNFS clonedb feature allows customers to set up their test database environment without really copying the production files into the new database environment. The files in the test database are created based on copy-on-write technology, so that only the blocks that are updated in the test database are ever written to the disk. This type of thin-provisioning reduces the amount of storage required for testing and development purposes. In addition to the reduced storage requirement, since the datafiles in the new test environment are not copied, setting up a test database is very quick and instantaneous.

Database cloning in a dNFS environment can be made from a rman full backup/image copy of all the relevant datafiles or from a storage snapshot. The database backup will be used as the backing store for the datafiles in the test database environment. Customers often take or required to take a full backup of their database and these files can be used to create the dnfs clonedb. By making the backup piece as the backing store the production data files are not touched by the test instances and the test instances don't compete for the same resources that the production database is using.
What do you need to do?
Following the few simple steps described below, one can clone a production database in a matter of few minutes. For the below example let us assume that your production database PROD1 is located at ORACLE_HOME=/u01/prod1/oracle. The rman backup of this production database is taken at RMAN_BACKUP_DIR=/u02/oracle/backup/prod1. The user is trying to clone a test database1 at ORACLE_HOME=/u03/test1/oracle and test database2 at ORACLE_HOME=/u03/test2/oracle

1.On your production database take a full rman hot/cold backup. If it is a hot backup make sure your production database is in archivelog mode and all the necessary archivelogs are saved and accessible to the new test database environment.
?To take a hot backup first put the database in hot backup mode.
alter database begin backup;
<Do a rman image copy/OS copy/Storage snapshot for each datafile>
alter database end backup;
?You can also run "backup as copy database format" rman command and get a copy of all your datafiles
Recovery Manager: Release 11.2.0.2.0 - Production on Wed Aug 18 10:25:35 2010
Copyright (c) 1982, 2009, Oracle and/or its affiliates. All rights reserved.
RMAN-06568: connected to target database: PROD1 (DBID=1985124264, not open)
RMAN-06008: connected to recovery catalog database
RMAN> backup as copy database format '/u02/oracle/backup/prod1/%U' ;
RMAN-03090: Starting backup at AUG 18 2010 10:25:37
RMAN-08030: allocated channel: ORA_DISK_1
RMAN-08500: channel ORA_DISK_1: SID=110 device type=DISK
RMAN-08580: channel ORA_DISK_1: starting datafile copy
RMAN-08522: input datafile file number=00001 name=/u01/prod1/oracle/dbs/tbs_01.f
RMAN-08586: output file name=/u02/oracle/backup/prod1/data_D-PROD1_I-1985124264_TS-SYSTEM_FNO-1_01lll1d6 tag=TAG20100818T102541 RECID=1 STAMP=727352775
RMAN-08581: channel ORA_DISK_1: datafile copy complete, elapsed time: 00:00:35

The above command creates a backup of all your datafiles in the location
"/u02/oracle/backup/prod1/"
2.Generate backup control file script from your production database by connecting with sysdba privileges and executing 'alter database backup controlfile to trace'. This will generate a trace file in your user_dump_dest directory which contains the create control file command.
3.Generate the testdb_create.sql using the script in the trace file.
SET ECHO ON
SET FEEDBACK 1
SET NUMWIDTH 10
SET LINESIZE 80
SET TRIMSPOOL ON
SET TAB OFF
SET PAGESIZE 100
STARTUP NOMOUNT PFILE='/u03/test1/oracle/dbs/testdb1.ora'
CREATE CONTROLFILE REUSE SET DATABASE "testdb1" RESETLOGS ARCHIVELOG
MAXLOGFILES 32 MAXLOGMEMBERS 2 MAXINSTANCES 1
MAXLOGHISTORY 908 LOGFILE
GROUP 1 '/u03/test1/oracle/dbs/t_1alog.f' SIZE 100M BLOCKSIZE 512,
GROUP 2 '/u03/test1/oracle/dbs/t_2alog.f' SIZE 100M BLOCKSIZE 512
DATAFILE
' /u02/oracle/backup/prod1/data_D-PROD1_I-1985124264_TS-SYSTEM_FNO-1_01lll1d6',...CHARACTER SET WE8DEC? ;

make sure you change the name of the database to the test db name and the log directories are pointing to the new testdb environment and use the image copy of the datafiles.
4.Copy spfile from your production db to the testdb environment and change the relevant parameters. In your testdb environment run the testdb_create.sql to create testdb control file. Execute dbms_dnfs.clonedb_renamefile(backup_file_name, new_data_filename) on each datafile in your test database.

SQL> declare
2 begin
3 dbms_dnfs.clonedb_renamefile('/u02/oracle/backup/prod1/data_D-LV1_I-1985124264_TS-SYSTEM_FNO-1_01lll1d6' , 'test1_tbs_01.f');
4 end;
5.PL/SQL procedure successfully completed.
Do this for each datafile in your testdb environment.
6.If you created your test clone from a hotbackup do a recover database recover database using backup controlfile until cancel;
The above command will ask for the archivelogs for the period when the backup was taken and input those file names.
SQL> recover database using backup controlfile until cancel ;
7.Open the database and start running your tests

SQL> alter database open resetlogs ;

At this point your testdb will be available for use and you will be able to read and write to the database. The testdb datafiles will not have allocated any storage for the datafiles. As you start running your test work load, inserts on the datafiles will allocate space on demand.

The above same procedure can be repeated for multiple test database instances using the same backup files. If you have a storage level snapshot taken on the datafile, the rman backup file names can be replaced with the storage snapshot names.

Since all the testdb clones are using the rman backup or the storage snapshot as the backend storage, availability of these files is critical for testdb to run. If the backup files become unavailable, you will see errors in the testdb database.

When it is time to destroy the testdb environment, all files in the test environment can be deleted without any impact on the production or backup environment. As this clonedb feature uses the backup piece as the backing storage, there is no pressure on the I/O subsystem that is servicing the production database. DBA's don't have to hunt for storage equivalent to the size of production database to set up a testdb environment. The testdb storage space usage grows at the speed at which the data is modified.

Work-in-progress - A perl script to automate the above steps is in progress. Using this script one will be able to clone a database by issuing a single command after setting few environment variables.

3. dnfs性能测试

外部链接：
New DNFS Performance Results
Monitoring Direct NFS with Oracle 11g and Solaris… pealing back the layers of the onion.
Direct NFS vs Kernel NFS bake-off with Oracle 11g and Solaris… and the winner is
Kernel NFS fights back… Oracle throughput matches Direct NFS with latest Solaris improvements

-fin-

Larry Ellison Appears In Iron Man 2

2010-05-11T20:35:00.001+08:00

埃里森出演钢铁侠2

Fluff: Larry Ellison in Iron Man 2
Silicon Valley shows up in 'Iron Man 2'
People: Larry Ellison fills starring 'Iron Man' role, sort of

-fin-

questions about database link

2009-12-24T17:11:00.000+08:00

questions about database link
dblink问题

Q1:
我现在有两个生产库，A库的部分表通过DBLINK提供给B库使用，B库的应用访问B库，并通过B库查询A库的这些DBLINK链接的表，现在有一个问题：
检查发现 B库的连接数比较多，进一步检查发现有50多个连接是来自A库，即使不用这些DBLINK的表，但连接还是建立的，通过DBLINK建立了8个表的连接，但实际上有50个连接，
是否应用不访问，那么DBLINK的表就没有连接？

A1:
1. 是长连接, 一般, 不用的不会释放
2. 跟会话有关, 不同会话之间不共享数据库连接
3. 可调整open_links参数, 控制打开的连接数

Q2:
奶奶的，会不会出问题？生产A 上通过DBLINK建立了 B库一个表，并建立同义词， A上一直往这个同义词上插入数据，导致如下的SQL建立的连接达到50多个，全部都是一样的，现在都停止插入了，但进程依然存在，刚才查了open_links参赛，都是4，但为什么会有这么多连接呢？怎么去控制？插入进程应该早就停止了并释放连接，但两个数据库之间的连接到底什么时候释放呢？

INSERT  INTO "SM_SEND_SM_LIST" ("SERIALNO","SERVICEID","SMCONTEN
T","SENDTARGET","PRIORITY","RCOMPLETETIMEBEGIN","RCOMPLETETIMEEN
D","RCOMPLETEHOURBEGIN","RCOMPLETEHOUREND","REQUESTTIME","ROADBY
","SENDTARGETDESC","FEEVALUE","LINKID","PAD1","PAD2","PAD3","PAD
4","PAD5") VALUES (:1,:2,:3,:4,:5,:6,:7,:8,:9,:10,:11,:12,:13,:1
4,:15,:16,:17,:18,:19)

刚才了解了一下，是应用程序往这个同义词插入数据，应用程序通过WEBLOGIC的连接池处理，是不是连接池的连接不释放，导致dblink连接不释放？
这玩意会不会增长到几百甚至更多？

A2:
1. 你的weblogic的连接池创建了50个会话连接到数据库上, 应用程序访问数据库, 会被随机分配一个连接, 可能是50个中的任何一个, 所以50个会话可能都被用到了, 所以数据库连接就会有50个
2. 程序退出, 连接池不释放连接, 会话依然存在, 所以数据库连接数不变
3. 只要会话数一定, 数据库连接数就一定, 所以不会涨到几百
4. open_links能够限制每个会话最大打开的数据库连接数, 没有系统全局的限制

Q3:
1，我的应用有7个呢，每个WEBLOGIC都设置有50个连接
2，检查发现大部分插入都是一个SQL，采用绑定变量方式；
3，难道就没有别的方式可以降低连接数量
4，Open_links这个参数好像对这个没有效果，每个进程肯定只建立一个连接，处理完成后进程消失，但连接不消失，而且从这个案例来说，不可能出现一个进程产生多个连接的现象

我考虑将这个表从B库迁移到A库，A库德插入可能是不同的进程，但B库扫描这个表的进程应该一直是一个，那么这样就应该只有一个连接了！

A3:
1. 你有7个weblogic, 每个的连接池有50个连接? 还是7个应用使用同一个weblogic, 一共有50个连接?
第一种情况: 如果7个weblogic, 连接池中每个连接都用到了你那个数据库连接, 那么应该最多看到350个连接
第二种情况, 如果连接池中每个连接都用到了数据库连接, 那么应该最多看到50个连接
上述任何一种情况, 如果不是每个连接都用到了数据库连接, 则看到的连接数应当更小

2. 跟这个没关, 即使用到了n个表, 连接也只有1个

3.
1)在本地数据库端只有open_links的限制
2)前面说过了回话退出时会释放连接
3)第一条指出了可能的最大连接数
4)alter session close database link可手工关闭连接, 但这么做没意义, 下次还会自动创建

4.用到了几个数据库连接, 建立的链接就是几个, 可建立的最大值由open_links确定
因为虽然程序(进程)退出了, 但连接池创建的会话不会退出, 所以连接不消失
因为连接池创建了50个连接(会话)到数据库,每个会话都用到了数据库连接, 所以你看到的连接数是50个

外部链接:
30 Managing a Distributed Database
OPEN_LINKS

-fin-

nmap ping scan

2009-12-24T16:35:00.004+08:00

nmap ping scan
nmap ping扫描

1. 现象
有3台服务器, 都开了iptables, 允许ICMP,SSH访问
server1: 192.168.18.95
server2: 192.168.51.72
server3: 192.168.51.71

nmap扫描服务器server1的ssh端口, 结果是通的

$ nmap server1 -p22

Starting Nmap 4.11 ( http://www.insecure.org/nmap/ ) at 2009-12-24 07:47 UTC
Interesting ports on server1 (192.168.18.95):
PORT   STATE SERVICE
22/tcp open  ssh

Nmap finished: 1 IP address (1 host up) scanned in 0.134 seconds

另一台server2扫描不通

$ nmap server2 -p22

Starting Nmap 4.11 ( http://www.insecure.org/nmap/ ) at 2009-12-24 07:48 UTC
Note: Host seems down. If it is really up, but blocking our ping probes, try -P0
Nmap finished: 1 IP address (0 hosts up) scanned in 3.001 seconds

然而ping却能ping通

$ ping server2 -c 3
PING server2 (192.168.51.72) 56(84) bytes of data.
64 bytes from server2 (192.168.51.72): icmp_seq=1 ttl=63 time=0.314 ms
64 bytes from server2 (192.168.51.72): icmp_seq=2 ttl=63 time=0.214 ms
64 bytes from server2 (192.168.51.72): icmp_seq=3 ttl=63 time=0.247 ms

--- server2 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 1999ms
rtt min/avg/max/mdev = 0.214/0.258/0.314/0.043 ms

SSH端口也可访问

$ ssh server2 uptime
username@server2's password:
 07:50:01 up 14 days,  8:02,  1 user,  load average: 0.00, 0.00, 0.00

将server2上的iptables关掉后, nmap扫描通过

# iptables -F

$ nmap server2 -p22

Starting Nmap 4.11 ( http://www.insecure.org/nmap/ ) at 2009-12-24 07:49 UTC
Interesting ports on server2 (192.168.51.72):
PORT   STATE SERVICE
22/tcp open  ssh

Nmap finished: 1 IP address (1 host up) scanned in 0.104 seconds

再打开server2上的iptables, 用root运行nmap扫描, 结果也是通的

# nmap server2 -p22

Starting Nmap 4.11 ( http://www.insecure.org/nmap/ ) at 2009-12-24 07:50 UTC
Interesting ports on server2 (192.168.51.72):
PORT   STATE SERVICE
22/tcp open  ssh

Nmap finished: 1 IP address (1 host up) scanned in 0.120 seconds

从server3用root向server2扫描也是通的

# nmap server2 -p22

Starting Nmap 4.11 ( http://www.insecure.org/nmap/ ) at 2009-12-24 08:10 UTC
Interesting ports on server2 (192.168.51.72):
PORT   STATE SERVICE
22/tcp open  ssh
MAC Address: 00:16:C0:A8:33:47 (Semtech)

Nmap finished: 1 IP address (1 host up) scanned in 0.141 seconds

server1的iptables设置:

*nat
...
-A PREROUTING -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 8080
-A OUTPUT -o lo -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 8080
...
*filter
...
-A INPUT -j RH-Firewall-1-INPUT
-A FORWARD -j RH-Firewall-1-INPUT
-A RH-Firewall-1-INPUT -i lo -j ACCEPT
-A RH-Firewall-1-INPUT -p icmp -m icmp --icmp-type any -j ACCEPT
-A RH-Firewall-1-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-A RH-Firewall-1-INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT
-A RH-Firewall-1-INPUT -p tcp -m tcp --dport 8080 -m state --state NEW -j ACCEPT
...
-A RH-Firewall-1-INPUT -j DROP

server2的iptables设置:

*filter
...
-A INPUT -j RH-Firewall-1-INPUT
-A FORWARD -j RH-Firewall-1-INPUT
-A RH-Firewall-1-INPUT -i lo -j ACCEPT
-A RH-Firewall-1-INPUT -p icmp -m icmp --icmp-type any -j ACCEPT
-A RH-Firewall-1-INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-A RH-Firewall-1-INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT
-A RH-Firewall-1-INPUT -p tcp -m tcp --dport 8080 -m state --state NEW -j ACCEPT
...
-A RH-Firewall-1-INPUT -j DROP

2. 调查
打开nmap调试信息和strace跟踪
1) server1:

$ nmap -v -d9 --packet-trace -n server1 -p22

Starting Nmap 4.11 ( http://www.insecure.org/nmap/ ) at 2009-12-24 07:52 UTC
The max # of sockets we are using is: 0
--------------- Timing report ---------------
  hostgroups: min 1, max 100000
  rtt-timeouts: init 1000, min 100, max 10000
  scan-delay: TCP 1000, UDP 1000
  parallelism: min 0, max 0
  max-retries: 10, host-timeout: 0
---------------------------------------------
READ selected for machine 192.168.18.95
Machine 192.168.18.95 MIGHT actually be listening on probe port 80
Hostupdate called for machine 192.168.18.95 state UNKNOWN/COMBO -> HOST_UP (trynum 0, dotimeadj: yes time: 727)
Timeout vals: srtt: -1 rttvar: -1 to: 1000000 delta 798 ==> srtt: 798 rttvar: 5000 to: 100000
Finished block: srtt: 798 rttvar: 5000 timeout: 100000 block_tries: 1 up_this_block: 1 down_this_block: 0 group_sz: 1
massping done:  num_hosts: 1  num_responses: 1
Initiating Connect() Scan against 192.168.18.95 [1 port] at 07:52
CONN (0.1040s) TCP localhost > 192.168.18.95:22 => Operation now in progress
**TIMING STATS**: IP, probes active/freshportsleft/retry_stack/outstanding/retranwait/onbench, cwnd/ccthresh/delay, timeout/srtt/rttvar/
   Groupstats (1/1 incomplete): 1/*/*/*/*/* 10.00/50/* 1000000/-1/-1
   192.168.18.95: 1/0/0/1/0/0 10.00/50/0 100000/798/5000
Discovered open port 22/tcp on 192.168.18.95
Changing ping technique for 192.168.18.95 to TCP
Timeout vals: srtt: 798 rttvar: 5000 to: 100000 delta -381 ==> srtt: 750 rttvar: 3845 to: 100000
Timeout vals: srtt: -1 rttvar: -1 to: 1000000 delta 417 ==> srtt: 417 rttvar: 5000 to: 100000
The Connect() Scan took 0.00s to scan 1 total ports.
Host 192.168.18.95 appears to be up ... good.
Interesting ports on 192.168.18.95:
Fetchfile found /usr/share/nmap/nmap-services

PORT   STATE SERVICE
22/tcp open  ssh
Final times for host: srtt: 750 rttvar: 3845  to: 100000

Nmap finished: 1 IP address (1 host up) scanned in 0.116 seconds

2) server2:

$ nmap -v -d9 --packet-trace -n server2 -p22

Starting Nmap 4.11 ( http://www.insecure.org/nmap/ ) at 2009-12-24 07:53 UTC
The max # of sockets we are using is: 0
--------------- Timing report ---------------
  hostgroups: min 1, max 100000
  rtt-timeouts: init 1000, min 100, max 10000
  scan-delay: TCP 1000, UDP 1000
  parallelism: min 0, max 0
  max-retries: 10, host-timeout: 0
---------------------------------------------
Finished block: srtt: -1 rttvar: -1 timeout: 1000000 block_tries: 2 up_this_block: 0 down_this_block: 0 group_sz: 1
massping done:  num_hosts: 1  num_responses: 0
Note: Host seems down. If it is really up, but blocking our ping probes, try -P0
Nmap finished: 1 IP address (0 hosts up) scanned in 3.002 seconds

与1)对比发现, nmap扫描server1时连接了server1的80端口

$ strace nmap server2 -p22
...
socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 3
fcntl(3, F_GETFL)                       = 0x2 (flags O_RDWR)
fcntl(3, F_SETFL, O_RDWR|O_NONBLOCK)    = 0
setsockopt(3, SOL_SOCKET, SO_LINGER, {onoff=1, linger=0}, 8) = 0
connect(3, {sa_family=AF_INET, sin_port=htons(80), sin_addr=inet_addr("192.168.51.72")}, 16) = -1 EINPROGRESS (Operation now in progress)
select(4, [3], [3], [3], {1, 0})        = 0 (Timeout)
close(3)                                = 0
socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 3
fcntl(3, F_GETFL)                       = 0x2 (flags O_RDWR)
fcntl(3, F_SETFL, O_RDWR|O_NONBLOCK)    = 0
setsockopt(3, SOL_SOCKET, SO_LINGER, {onoff=1, linger=0}, 8) = 0
connect(3, {sa_family=AF_INET, sin_port=htons(80), sin_addr=inet_addr("192.168.51.72")}, 16) = -1 EINPROGRESS (Operation now in progress)
select(4, [3], [3], [3], {1, 0})        = 0 (Timeout)
close(3)                                = 0
...
write(1, "Note: Host seems down. If it is "..., 81Note: Host seems down. If it is really up, but blocking our ping probes, try -P0
) = 81
...

连接server2的80端口, 2次超时(Timeout)后报错'Host seems down.', 然后退出

3) root用户扫描server2:

# nmap -v -d9 --packet-trace -n server2 -p22

Starting Nmap 4.11 ( http://www.insecure.org/nmap/ ) at 2009-12-24 07:53 UTC
The max # of sockets we are using is: 0
--------------- Timing report ---------------
  hostgroups: min 1, max 100000
  rtt-timeouts: init 1000, min 100, max 10000
  scan-delay: TCP 1000, UDP 1000
  parallelism: min 0, max 0
  max-retries: 10, host-timeout: 0
---------------------------------------------
Packet capture filter (device bond0): (icmp and dst host 192.168.11.18) or ((tcp or udp) and dst host 192.168.11.18 and ( dst port 34806 or dst port 34807 or dst port 34808 or dst port 34809 or dst port 34810))
SENT (0.0100s) ICMP 192.168.11.18 > 192.168.51.72 Echo request (type=8/code=0) ttl=52 id=10191 iplen=28
SENT (0.0100s) TCP 192.168.11.18:34810 > 192.168.51.72:80 A ttl=40 id=58342 iplen=40 seq=1674883102 win=1024 ack=2782179358
RCVD (0.0100s) ICMP 192.168.51.72 > 192.168.11.18 Echo reply (type=0/code=0) ttl=63 id=43696 iplen=28
We got a ping packet back from 192.168.51.72: id = 56906 seq = 21184 checksum = 52980
Hostupdate called for machine 192.168.51.72 state UNKNOWN/COMBO -> HOST_UP (trynum 0, dotimeadj: yes time: 446)
Timeout vals: srtt: -1 rttvar: -1 to: 1000000 delta 382 ==> srtt: 382 rttvar: 5000 to: 100000
Finished block: srtt: 382 rttvar: 5000 timeout: 100000 block_tries: 1 up_this_block: 1 down_this_block: 0 group_sz: 1
massping done:  num_hosts: 1  num_responses: 1
Initiating SYN Stealth Scan against 192.168.51.72 [1 port] at 07:53
Pcap filter: dst host 192.168.11.18 and (icmp or (tcp and (src host 192.168.51.72)))
Packet capture filter (device bond0): dst host 192.168.11.18 and (icmp or (tcp and (src host 192.168.51.72)))
SENT (0.1240s) TCP 192.168.11.18:34786 > 192.168.51.72:22 S ttl=51 id=27748 iplen=44 seq=3810228640 win=4096
**TIMING STATS**: IP, probes active/freshportsleft/retry_stack/outstanding/retranwait/onbench, cwnd/ccthresh/delay, timeout/srtt/rttvar/
   Groupstats (1/1 incomplete): 1/*/*/*/*/* 10.00/50/* 1000000/-1/-1
   192.168.51.72: 1/0/0/1/0/0 10.00/50/0 100000/382/5000
RCVD (0.1240s) TCP 192.168.51.72:22 > 192.168.11.18:34786 SA ttl=63 id=0 iplen=44 seq=3799856132 win=5840 ack=3810228641
Discovered open port 22/tcp on 192.168.51.72
Changing ping technique for 192.168.51.72 to TCP
Timeout vals: srtt: 382 rttvar: 5000 to: 100000 delta 44 ==> srtt: 387 rttvar: 3761 to: 100000
Timeout vals: srtt: -1 rttvar: -1 to: 1000000 delta 426 ==> srtt: 426 rttvar: 5000 to: 100000
The SYN Stealth Scan took 0.01s to scan 1 total ports.
Host 192.168.51.72 appears to be up ... good.
Interesting ports on 192.168.51.72:
Fetchfile found /usr/share/nmap/nmap-services

PORT   STATE SERVICE
22/tcp open  ssh
Final times for host: srtt: 387 rttvar: 3761  to: 100000

Nmap finished: 1 IP address (1 host up) scanned in 0.141 seconds
               Raw packets sent: 3 (112B) | Rcvd: 2 (92B)

发送ICMP echo request, 并向80端口发送TCP ACK包

# strace nmap server2 -p22
...
socket(PF_INET, SOCK_RAW, IPPROTO_RAW)  = 4
setsockopt(4, SOL_SOCKET, SO_BROADCAST, [1], 4) = 0
setsockopt(4, SOL_IP, IP_HDRINCL, [1], 4) = 0
socket(PF_INET, SOCK_RAW, IPPROTO_RAW)  = 5
setsockopt(5, SOL_SOCKET, SO_BROADCAST, [1], 4) = 0
setsockopt(5, SOL_IP, IP_HDRINCL, [1], 4) = 0
socket(PF_PACKET, SOCK_RAW, 768)        = 6
ioctl(6, SIOCGIFINDEX, {ifr_name="lo", ifr_index=1}) = 0
ioctl(6, SIOCGIFHWADDR, {ifr_name="bond0", ifr_hwaddr=00:14:22:12:39:2b}) = 0
ioctl(6, SIOCGIFINDEX, {ifr_name="bond0", ifr_index=5}) = 0
bind(6, {sa_family=AF_PACKET, proto=0x03, if5, pkttype=PACKET_HOST, addr(0)={0, }, 20) = 0
getsockopt(6, SOL_SOCKET, SO_ERROR, [17179869184], [4]) = 0
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 7
ioctl(7, SIOCGIFADDR, {ifr_name="bond0", ifr_addr={AF_INET, inet_addr("192.168.11.18")}}) = 0
ioctl(7, SIOCGIFNETMASK, {ifr_name="bond0", ifr_netmask={AF_INET, inet_addr("255.255.255.0")}}) = 0
close(7)                                = 0
brk(0x17164000)                         = 0x17164000
setsockopt(6, SOL_SOCKET, SO_ATTACH_FILTER, "\1\0\0\0\0\0\0\0\20)h\0\0\0\0\0", 16) = 0
fcntl(6, F_GETFL)                       = 0x2 (flags O_RDWR)
fcntl(6, F_SETFL, O_RDWR|O_NONBLOCK)    = 0
recvfrom(6, "\0\220\301\224x\377\177\0\0\220\301\224x\377\177\0\0p\301\224x\377\177\0\0\360\255\23\27\0\0\0"..., 1, MSG_TRUNC, NULL, NULL) = 114
recvfrom(6, "\0\220\301\224x\377\177\0\0\220\301\224x\377\177\0\0p\301\224x\377\177\0\0\360\255\23\27\0\0\0"..., 1, MSG_TRUNC, NULL, NULL) = 114
...
recvfrom(6, "\0\220\301\224x\377\177\0\0\220\301\224x\377\177\0\0p\301\224x\377\177\0\0\360\255\23\27\0\0\0"..., 1, MSG_TRUNC, NULL, NULL) = 146
recvfrom(6, 0x7fff7894c11f, 1, 32, 0, 0) = -1 EAGAIN (Resource temporarily unavailable)
fcntl(6, F_SETFL, O_RDWR)               = 0
setsockopt(6, SOL_SOCKET, SO_ATTACH_FILTER, "\231\0\224x\377\177\0\0\320\312\23\27\0\0\0\0", 16) = 0
sendto(5, "E\0\0\34\360Y\0\0-\1\335\334\300\250\v\22\300\2503H\10\0*\347\244\t)\17", 28, 0, {sa_family=AF_INET, sin_port=htons(0), sin_addr=inet_addr("192.168.51.72")}, 16) = 28
sendto(4, "E\0\0(\2\275\0\0;\6\275h\300\250\v\22\300\2503H\367\205\0Pu\303\312^\312\303\312^"..., 40, 0, {sa_family=AF_INET, sin_port=htons(80), sin_addr=inet_addr("192.168.51.72")}, 16) = 40
select(7, [6], NULL, NULL, {0, 20000})  = 1 (in [6], left {0, 20000})
recvfrom(6, "\0\24\"\0229+\0\23\200\6\301A\10\0E\0\0\34\252\274\0\0?\1\21z\300\2503H\300\250"..., 104, MSG_TRUNC, {sa_family=AF_PACKET, proto=0x800, if5, pkttype=PACKET_HOST, addr(6)={1, 00138006c141}, [18]) = 60
ioctl(6, SIOCGSTAMP, 0x7fff78948b20)    = 0
close(4)                                = 0
close(5)                                = 0
close(6)                                = 0
...

创建原始套接字, 向server2端口0和80发送数据包

4) server2关闭iptables:

$ strace nmap server2 -p22
...
socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 3
fcntl(3, F_GETFL)                       = 0x2 (flags O_RDWR)
fcntl(3, F_SETFL, O_RDWR|O_NONBLOCK)    = 0
setsockopt(3, SOL_SOCKET, SO_LINGER, {onoff=1, linger=0}, 8) = 0
connect(3, {sa_family=AF_INET, sin_port=htons(80), sin_addr=inet_addr("192.168.51.72")}, 16) = -1 EINPROGRESS (Operation now in progress)
select(4, [3], [3], [3], {1, 0})        = 2 (in [3], out [3], left {1, 0})
recvfrom(3, 0x7ffffaa7bed0, 255, 0, 0, 0) = -1 ECONNREFUSED (Connection refused)
close(3)                                = 0
...
socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 3
fcntl(3, F_GETFL)                       = 0x2 (flags O_RDWR)
fcntl(3, F_SETFL, O_RDWR|O_NONBLOCK)    = 0
setsockopt(3, SOL_SOCKET, SO_LINGER, {onoff=1, linger=0}, 8) = 0
connect(3, {sa_family=AF_INET, sin_port=htons(22), sin_addr=inet_addr("192.168.51.72")}, 16) = -1 EINPROGRESS (Operation now in progress)
select(4, [3], [3], [3], {0, 99000})    = 1 (out [3], left {0, 99000})
getsockopt(3, SOL_SOCKET, SO_ERROR, [0], [4]) = 0
sendto(3, "", 0, 0, NULL, 0)            = 0
getpeername(3, {sa_family=AF_INET, sin_port=htons(22), sin_addr=inet_addr("192.168.51.72")}, [17179869200]) = 0
getsockname(3, {sa_family=AF_INET, sin_port=htons(34707), sin_addr=inet_addr("192.168.11.18")}, [68719476752]) = 0
close(3)                                = 0
...

连接server2的80端口, 1次连接被拒绝(Connection refused)后, 不退出, 继续扫描22端口

5) root从server3扫描server2

# nmap -v -d9 --packet-trace -n server2 -p22

Starting Nmap 4.11 ( http://www.insecure.org/nmap/ ) at 2009-12-24 08:10 UTC
The max # of sockets we are using is: 0
--------------- Timing report ---------------
  hostgroups: min 1, max 100000
  rtt-timeouts: init 1000, min 100, max 10000
  scan-delay: TCP 1000, UDP 1000
  parallelism: min 0, max 0
  max-retries: 10, host-timeout: 0
---------------------------------------------
Initiating ARP Ping Scan against 192.168.51.72 [1 port] at 08:10
Pcap filter: arp and ether dst host 00:16:C0:A8:33:46
Packet capture filter (device eth0): arp and ether dst host 00:16:C0:A8:33:46
SENT (0.0140s) ARP who-has 192.168.51.72 tell 192.168.51.71
**TIMING STATS**: IP, probes active/freshportsleft/retry_stack/outstanding/retranwait/onbench, cwnd/ccthresh/delay, timeout/srtt/rttvar/
   Groupstats (1/1 incomplete): 1/*/*/*/*/* 10.00/50/* 100000/-1/-1
   192.168.51.72: 1/0/0/1/0/0 10.00/50/0 100000/-1/-1
RCVD (0.0150s) ARP reply 192.168.51.72 is-at 00:16:C0:A8:33:47
Timeout vals: srtt: -1 rttvar: -1 to: 100000 delta 364 ==> srtt: 364 rttvar: 5000 to: 100000
Timeout vals: srtt: -1 rttvar: -1 to: 100000 delta 364 ==> srtt: 364 rttvar: 5000 to: 100000
The ARP Ping Scan took 0.01s to scan 1 total hosts.
Initiating SYN Stealth Scan against 192.168.51.72 [1 port] at 08:10
Pcap filter: dst host 192.168.51.71 and (icmp or (tcp and (src host 192.168.51.72)))
Packet capture filter (device eth0): dst host 192.168.51.71 and (icmp or (tcp and (src host 192.168.51.72)))
SENT (0.0380s) TCP 192.168.51.71:35569 > 192.168.51.72:22 S ttl=57 id=4614 iplen=44 seq=1639929488 win=2048
**TIMING STATS**: IP, probes active/freshportsleft/retry_stack/outstanding/retranwait/onbench, cwnd/ccthresh/delay, timeout/srtt/rttvar/
   Groupstats (1/1 incomplete): 1/*/*/*/*/* 10.00/50/* 1000000/-1/-1
   192.168.51.72: 1/0/0/1/0/0 10.00/50/0 100000/364/5000
RCVD (0.0390s) TCP 192.168.51.72:22 > 192.168.51.71:35569 SA ttl=64 id=0 iplen=44 seq=474347775 win=5840 ack=1639929489
Discovered open port 22/tcp on 192.168.51.72
Changing ping technique for 192.168.51.72 to TCP
Timeout vals: srtt: 364 rttvar: 5000 to: 100000 delta 9 ==> srtt: 365 rttvar: 3752 to: 100000
Timeout vals: srtt: -1 rttvar: -1 to: 1000000 delta 373 ==> srtt: 373 rttvar: 5000 to: 100000
The SYN Stealth Scan took 0.01s to scan 1 total ports.
Fetchfile found /usr/share/nmap/nmap-mac-prefixes

Host 192.168.51.72 appears to be up ... good.
Interesting ports on 192.168.51.72:
Fetchfile found /usr/share/nmap/nmap-services

PORT   STATE SERVICE
22/tcp open  ssh
MAC Address: 00:16:C0:A8:33:47 (Semtech)
Final times for host: srtt: 365 rttvar: 3752  to: 100000

Nmap finished: 1 IP address (1 host up) scanned in 0.142 seconds
               Raw packets sent: 2 (86B) | Rcvd: 2 (86B)

发送ARP扫描

3. 分析
查看nmap手册

$ man nmap
...
HOST DISCOVERY
One of the very first steps in any network reconnaissance mission is to reduce a (sometimes huge) set of IP ranges into a list of active or interesting hosts. Scanning every port of every single IP address is slow and usually unnecessary. Of course what makes a host interesting depends greatly on the scan purposes. Network administrators may only be interested in hosts running a certain service, while security auditors may care about every single device with an IP address. An administrator may be comfortable using just an ICMP ping to locate hosts on his internal network, while an external penetration tester may use a diverse set of dozens of probes in an attempt to evade firewall restrictions.

Because host discovery needs are so diverse, Nmap offers a wide variety of options for customizing the techniques used. Host discovery is sometimes called ping scan, but it goes well beyond the simple ICMP echo request packets associated with the ubiquitous ping tool. Users can skip the ping step entirely with a list scan (-sL) or by disabling ping (-P0), or engage the network with arbitrary combinations of multi-port TCP SYN/ACK, UDP, and ICMP probes. The goal of these probes is to solicit responses which demonstrate that an IP address is actually active (is being used by a host or network device). On many networks, only a small percentage of IP addresses are active at any given time. This is particularly common with RFC1918-blessed private address space such as 10.0.0.0/8. That network has 16 million IPs, but I have seen it used by companies with less than a thousand machines. Host discovery can find those machines in a sparsely allocated sea of IP addresses.

If no host discovery options are given, Nmap sends a TCP ACK packet destined for port 80 and an ICMP Echo Request query to each target machine. An exception to this is that an ARP scan is used for any targets which are on a local ethernet network. For unprivileged UNIX shell users, a SYN packet is sent instead of the ack using the connect() system call. These defaults are equivalent to the -PA -PE options. This host discovery is often sufficient when scanning local networks, but a more comprehensive set of discovery probes is recommended for security auditing.
...
-sP (Ping Scan)
This option tells Nmap to only perform a ping scan (host discovery), then print out the available hosts that responded to the scan. No further testing (such as port scanning or OS detection) is performed. This is one step more intrusive than the list scan, and can often be used for the same purposes. It allows light reconnaissance of a target network without attracting much attention. Knowing how many hosts are up is more valuable to attackers than the list provided by list scan of every single IP and host name.

Systems administrators often find this option valuable as well. It can easily be used to count available machines on a network or monitor server availability. This is often called a ping sweep, and is more reliable than pinging the broadcast address because many hosts do not reply to broadcast queries.

The -sP option sends an ICMP echo request and a TCP packet to port 80 by default. When executed by an unprivileged user, a SYN packet is sent (using a connect() call) to port 80 on the target. When a privileged user tries to scan targets on a local ethernet network, ARP requests (-PR) are used unless --send-ip was specified. The -sP option can be combined with any of the discovery probe types (the -P* options, excluding -P0) for greater flexibility. If any of those probe type and port number options are used, the default probes (ACK and echo request) are overridden. When strict firewalls are in place between the source host running Nmap and the target network, using those advanced techniques is recommended. Otherwise hosts could be missed when the firewall drops probes or their responses.
...

nmap在检测服务和端口之前, 默认首先检测主机是否是活动的, 这被称为主机发现(Host Disvocery)或ping扫描(ping scan)
如果主机发现不通过, 则不进行后面的端口扫描
主机发现的方式有多种, 默认方式是向主机80端口发送一个TCP ACK包, 并发送ICMP 回显请求(Echo Request)查询, 如在同一局域网则使用ARP扫描. 对于UNIX下非特权用户, 不发送ack包, 而是使用connect()系统调用发送一个SYN包.

4. 结论
1) 特权用户才可以建立原始套接字, 使用ARP, ICMP, ACK扫描

2) 普通用户运行nmap扫描server1:
nmap使用connect向80端口发送SYN, 因为server1上启动了tomcat服务, 80端口可访问, 所以主机发现成功通过

3) 普通用户运行nmap扫描打开iptables的server2:
nmap使用connect向80端口发送SYN, server2上iptables不允许访问80端口, 造成扫描超时错误, nmap认为主机不可达, 所以主机发现失败

4) 普通用户运行nmap扫描关闭iptables的server2:
nmap使用connect向80端口发送SYN, server2上没有监听80端口的服务, 造成连接被拒绝, nmap认为主机可到达, 所以主机发现通过

5) 特权用户运行nmap扫描打开iptables的server2:
nmap使用原始套接字发出ICMP回显请求, server2上iptables允许ICMP, 所以主机发现能通过

6) 普通用户ping命令扫描打开iptables的server2:
因为ping命令拥有setuid root权限, 运行时具有root权限, 可以创建原始套接字用ICMP扫描, 所以能ping通主机

$ ls -l $(which ping)
-rwsr-xr-x 1 root root 37280 Mar 14  2007 /bin/ping

6) 特权用户从server3扫描打开iptables的server2:
因为server2和server3处于同一子网, 所以使用ARP扫描, ARP属于第2层数据链路层, iptables无法过滤, 所以主机发现能通过

外部链接:
Chapter 15. Nmap Reference Guide
The Art of Port Scanning - by Fyodor

-fin-

How to download oracle patches from CLI

2009-12-17T00:18:00.007+08:00

How to download oracle patches from CLI
如何从命令行下载Oracle补丁

Oracle的FTP下载站点已于2009年11月6号关闭, 不再提供服务, 下载补丁只能通过My Oracle Support网站

$ ftp updates.oracle.com
Connected to updates.oraclegha.com.
421-*********************** Downtime Notice ************************
421-
421-This service was retired as of November 06, 2009.
421-
421-****************************************************************
421
ftp>

其实可以用命令行工具以HTTPS方式下载补丁
补丁的URL地址格式是:

https://updates.oracle.com/Orion/Services/download/?aru=<ARU编号>
ARU(Automated Release Updates)编号可在My Oracle Support网站上查到
1) 访问My Oracle Support网站

2) 登录

3) 选择"Patches & Updates"

4) 查找补丁

5) 单击选择该行, 弹出菜单, 选择"View Read Me"

6) 在说明文档的URL地址栏中找到ARU编号

比如,得到10.2.0.4 Patch Set的下载地址是:

https://updates.oracle.com/Orion/Services/download/?aru=10029612

然后用wget工具下载即可, 在命令行中还需提供登录用户名和密码, 如:

wget --http-user="myusername" --http-password="mypassword" "https://updates.oracle.com/Orion/Services/download/?aru=10029612"

对有密码保护的补丁, URL中再加上patch_password参数, 如:

https://updates.oracle.com/Orion/Services/download/?aru=12345678&patch_password=nosuchpasswd

wget常见问题:
1. File name too long
报错"File name too long", 无法写文件

Resolving oracle-updates.oracle.com... 68.142.116.74, 68.142.116.73
Connecting to oracle-updates.oracle.com|68.142.116.74|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 111,654 (109K) [application/zip]
p8524113_10204_Linux-x86-64.zip?download_url=http:%2F%2Fupdates.oracle.com%2FARULink%2FDownload%2Fprocess_form&userid=o-wenxie@motorola.com&email=wenxie@motorola.com&record_download_url=http:%2F%2Fupdates.oracle.com%2FARULink%2FPatchDownload%2Fpack_stats&timestamp=1260979159&patch_password=NO_PASS&context=A@10+H@aaru203.oracle.com+P@&aru=11423647&patch_file=p8524113_10204_Linux-x86-64.zip&ip_address=123.127.66.13&file_id=27511963&use_proxy=false&id=ZWtXM1l3WXFjVUlEZ2YwVDFUZ1ZOQTpHOWEvMlhXSTlRTDJRL2RKbXMvejJB: File name too long

Cannot write to `p8524113_10204_Linux-x86-64.zip?download_url=http:%2F%2Fupdates.oracle.com%2FARULink%2FDownload%2Fprocess_form&userid=o-wenxie@motorola.com&email=wenxie@motorola.com&record_download_url=http:%2F%2Fupdates.oracle.com%2FARULink%2FPatchDownload%2Fpack_stats&timestamp=1260979159&patch_password=NO_PASS&context=A@10+H@aaru203.oracle.com+P@&aru=11423647&patch_file=p8524113_10204_Linux-x86-64.zip&ip_address=123.127.66.13&file_id=27511963&use_proxy=false&id=ZWtXM1l3WXFjVUlEZ2YwVDFUZ1ZOQTpHOWEvMlhXSTlRTDJRL2RKbXMvejJB' (File name too long).

需要指定下载文件名, 该文件名可从wget输出中得到
文件名格式为:

p<补丁编号>_<数据库版本>_<操作系统>.zip

如:

p6810189_10204_Linux-x86-64.zip

也可在My Oracle Support查到
1) 访问My Oracle Support网站
2) 登录
3) 选择"Patches & Updates"
4) 查找补丁
5) 点该行开头的补丁编号

6) 显示出补丁详细信息, 单击"Show File Details"

7) 显示出文件名

在wget命令中加-O选项指定下载文件名:

wget --http-user="myusername" --http-password="mypassword" -O "p6810189_10204_Linux-x86-64.zip" "https://updates.oracle.com/Orion/Services/download/?aru=11423647"

2. 加-c选项不能断点续传
报错304 Not Modified

Location: https://updates.oracle.com/Orion/Services/download/?aru=10029612 [following]
--15:59:04--  https://updates.oracle.com/Orion/Services/download/?aru=10029612
Connecting to updates.oracle.com|141.146.44.51|:443... connected.
HTTP request sent, awaiting response... 302 Found
Location: http://oracle-updates.oracle.com/ARUConnect/p6810189_10204_Linux-x86-64.zip?download_url=http%3A%2F%2Fupdates.oracle.com%2FARULink%2FDownload%2Fprocess_form&userid=o-wenxie%40motorola.com&record_download_url=http%3A%2F%2Fupdates.oracle.com%2FARULink%2FPatchDownload%2Fpack_stats&email=wenxie%40motorola.com&timestamp=1260979145&patch_password=NO_PASS&context=A%4010%2BH%40aaru204.oracle.com%2BP%40&ip_address=123.127.66.8&patch_file=p6810189_10204_Linux-x86-64.zip&aru=10029612&file_id=22503408&use_proxy=false&id=QUViSEhzNWVGZFlpMGZvVjBlRnZwQTppeWJXRk9Mc2xiVitPU2F1UTNrd21R [following]
--15:59:05--  http://oracle-updates.oracle.com/ARUConnect/p6810189_10204_Linux-x86-64.zip?download_url=http%3A%2F%2Fupdates.oracle.com%2FARULink%2FDownload%2Fprocess_form&userid=o-wenxie%40motorola.com&record_download_url=http%3A%2F%2Fupdates.oracle.com%2FARULink%2FPatchDownload%2Fpack_stats&email=wenxie%40motorola.com&timestamp=1260979145&patch_password=NO_PASS&context=A%4010%2BH%40aaru204.oracle.com%2BP%40&ip_address=123.127.66.8&patch_file=p6810189_10204_Linux-x86-64.zip&aru=10029612&file_id=22503408&use_proxy=false&id=QUViSEhzNWVGZFlpMGZvVjBlRnZwQTppeWJXRk9Mc2xiVitPU2F1UTNrd21R
Resolving oracle-updates.oracle.com... 68.142.116.74, 68.142.116.73
Connecting to oracle-updates.oracle.com|68.142.116.74|:80... connected.
HTTP request sent, awaiting response... 304 Not Modified
15:59:05 ERROR 304: Not Modified.

同样需要指定下载文件名:

wget --http-user="myusername" --http-password="mypassword" -c -O "p6810189_10204_Linux-x86-64.zip" "https://updates.oracle.com/Orion/Services/download/?aru=11423647"

3. HTTPS证书出错
加上--no-check-certificate选项

外部链接:
Classic MetaLink and My Oracle Support Transition Information Page
Retrieving Oracle patches with wget
Automated 'wget' patch downloads: issue resolution

-fin-

trim spaces from a shell string

2009-11-23T16:07:00.003+08:00

how to remove leading and trailing spaces from a shell variable?
如何删除shell字符串变量中的前缀的和后缀的空格?

var="   hello world!    "
echo "var=\"$var\""

var="   hello world!    "

1. echo

newvar=$(echo $var)
echo "newvar=\"$newvar\""

newvar="hello world!"

2. Parameter Substitution

newvar=${var#"${var%%[![:blank:]]*}"}
newvar=${newvar%"${newvar##*[![:blank:]]}"}
echo "newvar=\"$newvar\""

newvar="hello world!"

3. Pattern Matching

shopt -s extglob
newvar=${var##+( )}
newvar=${newvar%%+( )}
shopt -u extglob
echo "newvar=\"$newvar\""

newvar="hello world!"

4. sed

newvar=$(echo "$var"|sed 's/\(^[[:blank:]]\+\|[[:blank:]]\+$\)//g')
echo "newvar=\"$newvar\""

newvar="hello world!"

5. awk

newvar=$(echo "$var"|awk 'gsub(/^[[:blank:]]+|[[:blank:]]+$/,"")')
echo "newvar=\"$newvar\""

newvar="hello world!"

-fin-

how to resync a broken master-master replication

2009-10-30T17:21:00.000+08:00

how to resync a broken master-master replication

[still in draft]

db01:3306 and db03:3306 are setting up with master-master replicaition.

in db03 the slave process is accidentally stopped for quite a long time that caused by a wrong configuration, after resolving the problem, dual master replication seems to start working, but unfortunately binary logs were delete older than x days, many tables has got out of date or even lost in db03, recreating the slave db is not acceptable, we have to find a more efficient way to make thier data resync again.

1. stop replication from db03 to db01
login db01

show slave status\G
stop slave;
show slave status\G

2. dump sql script for recreating table structure later
login db01

DB_LIST=$(mysql -u root --batch --skip-column-names -e 'show databases;'|sed '/^information_schema$\|^mysql$\|^test$/d'|tr '\n' ' ')
mysqldump -u root --no-data --skip-add-drop-table --databases $DB_LIST >$HOME/db01-3306-db.sql

DB_LIST=$(mysql -u root --batch --skip-column-names -e 'show databases;'|sed '/^information_schema$\|^mysql$\|^test$/d'|tr '\n' ' ')
mysqldump -u root --no-data --skip-add-drop-table --databases $DB_LIST >$HOME/db03-3306-db.sql

3. recreate lost tables
login db03

scp db01:db01-3306-db.sql $HOME/
mysql -u root --force <$HOME/db01-3306-db.sql

scp db03:db03-3306-db.sql $HOME/
mysql -u root --force <$HOME/db03-3306-db.sql

4. solve slave sql issues in db03, if any
login db03

show slave status\G
set global sql_slave_skip_counter=1;
start slave;
show slave status\G

now db01 and db03 are actually working in master-slave replication.

5. check data consistency between db01 and db03
run mk-table-checksum against db01, it will execute sql queries to calculate crc checksum of tables in db01, then in db02 the same queries will run too, and calculate checksum in db02, finally we will get checksums of both databases.

mk-table-checksum --ignore-databases=mysql,test --replicate mysql.checksum --empty-replicate-table --create-replicate-table h=db01,P=3306,u=webuser

6. sync table records from db01 to db03
run mk-table-sync with dry-run or print option, see what will happen in db03, it will not actually take any actions.

mk-table-sync --dry-run --ignore-databases=mysql,test --sync-to-master --replicate=mysql.checksum h=db03,P=3306,u=webuser
mk-table-sync --print --ignore-databases=mysql,test --sync-to-master --replicate=mysql.checksum h=db03,P=3306,u=webuser

actually sync table data

mk-table-sync --execute --ignore-databases=mysql,test --sync-to-master --replicate=mysql.checksum h=db03,P=3306,u=webuser

7. known issue
"Can't make changes on the master: no unique index exists at /usr/bin/mk-table-sync line 3775. while doing dbtest.t"
solution: create unique index on the table
it is not a perfect solution, because there may be no suitable columns for defining an unique index, or there may be duplicate records in the table.

8. check to see if all data in both databases are identical

mk-table-checksum --ignore-databases=mysql,test -P 3306 -u webuser db01 db03|mk-checksum-filter

9. start replication from db03 to db01
login db01

show slave status\G
start slave;
show slave status\G

10. external links:

mk-table-checksum
mk-table-sync
How to sync tables in master-master MySQL replication
MySQL Master-Master replication table sync
Maatkit Options for Restoring a Slave or Master
Tip of the Day — Checking Your Tables — Part I
Sync up a MySQL slave with mk-table-sync
mk-table-sync and small tables

-fin-

how to generate sequence in bash

2009-10-27T12:19:00.000+08:00

how to generate sequence in bash
BASH脚本如何产生序号

1. BASH FOR循环结构(Looping Constructs)

$ for ((a=1;a<=5;a++));do echo $a;done
1
2
3
4
5

这种用法只能在FOR循环中使用

2. BASH大括号扩展机制(Brace Expansion)

$ echo {1..10}
1 2 3 4 5 6 7 8 9 10
$ echo {z..a}
z y x w v u t s r q p o n m l k j i h g f e d c b a

bash第4版增加了功能,可指定步长

开头补零:

$ printf "%02d " {6..12}
06 07 08 09 10 11 12

3. seq命令
seq命令是属于coreutils包的一个工具, 用于生成一组数字序列号, 如:

$ seq 1 2 10
1
3
5
7
9

开头补零:

$ seq -f %02.0f 6 12
06
07
08
09
10
11
12

4. awk

$ awk 'BEGIN {for (i=1;i<=5;i++) print i}'
1
2
3
4
5

5. jot
BSD系统中提供, 可以产生序列,随机数
jot - UNIX Power Tools, 2nd Edition
Using BSD jot
jot (BSD)
JOT(1)

-fin-

monitoring io statistics by process

2009-10-26T15:41:00.003+08:00

monitoring io statistics by process
监控进程的IO统计信息

1. I/O accounting
Linux内核2.6.20增加了IO审计功能, 要使用这个功能需要首先在编译内核时打开TASK_DELAY_ACCT和TASK_IO_ACCOUNTING参数
RedHat Linux在内核2.6.18-144之后也加入了该功能, 并在RHEL5.4版本(kernel 2.6.18-164)中发布, 可以直接使用

一些基于IO审计的监控工具:
1)IOtop
Iotop is a Python program with a top like UI used to show of behalf of which process is the I / O going on. It requires Python ≥ 2.5 and a Linux kernel ≥ 2.6.20 with the TASK_DELAY_ACCT and TASK_IO_ACCOUNTING options enabled. It requires Python ≥ 2.5 and ≥ Linux kernel 2.6.20 with the TASK_DELAY_ACCT and TASK_IO_ACCOUNTING options enabled.

2)pidstat(SYSSTAT工具之一)
Report I/O statistics (kernels 2.6.20 and later only)

3)Dstat: Versatile resource statistics tool
0.6.7 版本后支持显示topio,topbio

4)htop - an interactive process viewer for Linux
显示RBYTES,WBYTES,IO_RATE等列

5)collectl - Process I/O Stats
2.4.0版本之后支持显示io统计信息, 见Process I/O Stats

外部链接
Red Hat backported I/O accounting to RHEL5
Red Hat Enterprise Linux 5.4 - Release Notes

2. block_dump
设置内核参数block_dump为非零值, 内核将报告所有磁盘读写操作和脏块刷新的信息

下载iodump脚本, 用于分析处理内核报告

wget http://maatkit.googlecode.com/svn/trunk/util/iodump

打开block_dump

echo 1 > /proc/sys/vm/block_dump

通过iodump分析dmesg输出的内核报告

while true; do sleep 1; dmesg -c; done | perl iodump

运行一会儿, 按Ctrl-c停止输出, 得到类似下面的结果

# while true; do sleep 1; dmesg -c; done | perl iodump
# Caught SIGINT.
TASK                   PID      TOTAL       READ      WRITE      DIRTY DEVICES
kjournald              478       1294          0       1294          0 sda2
pdflush              21422        857          0        857          0 sda5, sdb6
kjournald             1752        365          0        365          0 sda5
kjournald             1758        174          0        174          0 sdb6
kjournald             1756         52          0         52          0 sdb5
syslogd              26752         28          0         28          0 sdb6
firefox              12811         27         27          0          0 sdb5
perl                  3003         25         25          0          0 sda2
process_perfdat       3026         12         12          0          0 sda5
bash                 14749          3          3          0          0 sda2
firefox              12654          3          3          0          0 sdb5
bash                  3004          2          2          0          0 sda2
bash                  3017          1          1          0          0 sda2
squid                25555          1          1          0          0 sda2
check_nrpe            3001          1          1          0          0 sda2

最后关闭block_dump

echo 0 > /proc/sys/vm/block_dump

外部链接:
block_dump
How to find per-process I/O statistics on Linux
Monitoring filesystem activity under Linux with block_dump
How can I record what process or kernel activity is using the disk in GNU/Linux?
How to identify what processes are generating IO Wait load.

3. blktrace
支持block trace的内核版本:
– Patch for Linux 2.6.14rc3
(or later, up to 2.6.17)
– Linux 2.6.17 (or later) – built in

安装

yum -y install blktrace

挂载debugfs文件系统

mount -t debugfs debugfs /sys/kernel/debug

运行blktrace

blktrace -d /dev/sda2 -o - | blkparse -i - -s

运行一段时间后, CTRL-C退出, 显示统计信息:

...
...
...
  8,2    0     1576     0.618648628   478  U   N [kjournald] 1
  8,2    0     1577     0.618669180     0  C   W 375869 + 128 [0]
  8,2    0     1578     0.618763920   478  Q   W 376045 + 8 [kjournald]
  8,2    0     1579     0.618765468   478  G   W 376045 + 8 [kjournald]
  8,2    0     1580     0.618766125   478  P   N [kjournald]
  8,2    0     1581     0.618766440   478  I   W 376045 + 8 [kjournald]
  8,2    0     1582     0.618767278   478  U   N [kjournald] 1
  8,2    0     1583     0.618768643   478  D   W 376045 + 8 [kjournald]
  8,2    0     1584     0.618824993     0  C   W 376045 + 8 [0]
blktrace (5609)
 Reads Queued:           0,        0KiB  Writes Queued:           0,        0KiB
 Read Dispatches:        0,        0KiB  Write Dispatches:        2,        8KiB
 Reads Requeued:         0               Writes Requeued:         0
 Reads Completed:        0,        0KiB  Writes Completed:        2,       12KiB
 Read Merges:            0,        0KiB  Write Merges:            0,        0KiB
 IO unplugs:             0               Timer unplugs:           0
 Allocation wait:        0               Allocation wait:         0
 Dispatch wait:          0               Dispatch wait:           0
 Completion wait:        0               Completion wait:         0
kjournald (478)
 Reads Queued:           0,        0KiB  Writes Queued:         330,    1,320KiB
 Read Dispatches:        0,        0KiB  Write Dispatches:       42,    1,320KiB
 Reads Requeued:         0               Writes Requeued:         0
 Reads Completed:        0,        0KiB  Writes Completed:        4,       76KiB
 Read Merges:            0,        0KiB  Write Merges:          288,    1,152KiB
 IO unplugs:            12               Timer unplugs:           0
 Allocation wait:        0               Allocation wait:         0
 Dispatch wait:          0               Dispatch wait:           0
 Completion wait:        0               Completion wait:         0
nagios (5574)
 Reads Queued:           0,        0KiB  Writes Queued:           0,        0KiB
 Read Dispatches:        0,        0KiB  Write Dispatches:        0,        0KiB
 Reads Requeued:         0               Writes Requeued:         0
 Reads Completed:        0,        0KiB  Writes Completed:        1,       16KiB
 Read Merges:            0,        0KiB  Write Merges:            0,        0KiB
 IO unplugs:             0               Timer unplugs:           0
 Allocation wait:        0               Allocation wait:         0
 Dispatch wait:          0               Dispatch wait:           0
 Completion wait:        0               Completion wait:         0
pdflush (776)
 Reads Queued:           0,        0KiB  Writes Queued:         126,      504KiB
 Read Dispatches:        0,        0KiB  Write Dispatches:       33,      132KiB
 Reads Requeued:         0               Writes Requeued:         0
 Reads Completed:        0,        0KiB  Writes Completed:       33,      136KiB
 Read Merges:            0,        0KiB  Write Merges:            1,        4KiB
 IO unplugs:             0               Timer unplugs:           0
 Allocation wait:        0               Allocation wait:         0
 Dispatch wait:          0               Dispatch wait:           0
 Completion wait:        0               Completion wait:         0
swapper (0)
 Reads Queued:           0,        0KiB  Writes Queued:           0,        0KiB
 Read Dispatches:        0,        0KiB  Write Dispatches:      131,      536KiB
 Reads Requeued:         0               Writes Requeued:         0
 Reads Completed:        0,        0KiB  Writes Completed:      233,    2,404KiB
 Read Merges:            0,        0KiB  Write Merges:            0,        0KiB
 IO unplugs:             0               Timer unplugs:           0
 Allocation wait:        0               Allocation wait:         0
 Dispatch wait:          0               Dispatch wait:           0
 Completion wait:        0               Completion wait:         0

CPU0 (8,2):
 Reads Queued:           0,        0KiB  Writes Queued:         456,    1,824KiB
 Read Dispatches:        0,        0KiB  Write Dispatches:      208,    1,996KiB
 Reads Requeued:         0               Writes Requeued:         0
 Reads Completed:        0,        0KiB  Writes Completed:      273,    2,644KiB
 Read Merges:            0,        0KiB  Write Merges:          289,    1,156KiB
 Read depth:             0               Write depth:            20
 IO unplugs:            11               Timer unplugs:           0
CPU2 (8,2):
 Reads Queued:           0,        0KiB  Writes Queued:           0,        0KiB
 Read Dispatches:        0,        0KiB  Write Dispatches:        0,        0KiB
 Reads Requeued:         0               Writes Requeued:         0
 Reads Completed:        0,        0KiB  Writes Completed:        0,        0KiB
 Read Merges:            0,        0KiB  Write Merges:            0,        0KiB
 Read depth:             0               Write depth:            20
 IO unplugs:             1               Timer unplugs:           0

Total (8,2):
 Reads Queued:           0,        0KiB  Writes Queued:         456,    1,824KiB
 Read Dispatches:        0,        0KiB  Write Dispatches:      208,    1,996KiB
 Reads Requeued:         0               Writes Requeued:         0
 Reads Completed:        0,        0KiB  Writes Completed:      273,    2,644KiB
 Read Merges:            0,        0KiB  Write Merges:          289,    1,156KiB
 IO unplugs:            12               Timer unplugs:           0

Throughput (R/W): 0KiB/s / 4,278KiB/s
Events (8,2): 1,585 entries
Skips: 0 forward (0 -   0.0%)

最后卸载debug文件系统

umount /sys/kernel/debug

外部链接:
Block I/O Layer Tracing: blktrace
blktrace User Guide
Tracing I/O usage on Linux

4. atop
对内核源码打补丁, 能够支持显示每个进程的IO

5. systemtap
例子见http://bpineau.livejournal.com/

6. DTrace
Solaris专用
iotop脚本见Top Ten DTrace (D) Scripts
iotop - display top disk I/O events by process.

7. 其它
pio and topio: for solaris,hp-ux,windows
Psio + other disk I/O by process tools: for solaris only

-fin-

setting up pitrtools

2009-10-20T18:35:00.001+08:00

setting up pitrtools

pitrtools也是一种管理postgresql备库的工具集, 由2个python编写的小脚本组成
cmd_archiver, 调用rsync复制归档日志文件, 支持向多个备用库服务器传输文件, 这里简单介绍了传输文件的流程PITRTools: Multiple slave support
cmd_standby, 将数据库从主库备份到备库, 调用pg_standby在备库上应用归档日志

1. 测试环境:
primary(master) db: pgdb01
standby(slave) db: pgdb02
db version: Postgresql 8.3.7
os version: CentOS 5.3

2. 配置ssh登录
略, 见前篇博客setting up postgresql warm standby

3. 安装Pitrtools
登录pgdb01,pgdb02, 下载安装

wget -O - http://files.commandprompt.com/pitrtools/pitrtools-1.2.tar.bz2|tar -C $HOME -jxf -
chmod +x $HOME/pitrtools-1.2/{cmd_archiver,cmd_standby}

4. 主库配置archiver

编辑配置文件

cat >$HOME/pitrtools-1.2/cmd_archiver.ini <<EOF
[DEFAULT]
state: online
pgdata: /var/lib/pgsql/data
r_archivedir: /var/lib/pgsql/archive
l_archivedir: /var/lib/pgsql/archive
rsync_bin: /usr/bin/rsync
rsync_version = 2
slaves: pgdb02
user: postgres
timeout: 10
notify_ok: echo OK
notify_warning:  echo WARNING
notify_critical: echo CRITICAL
debug: on
ssh_debug: off
EOF

初始化建归档目录

$HOME/pitrtools-1.2/cmd_archiver -C $HOME/pitrtools-1.2/cmd_archiver.ini -I

We are initializing queues, one moment.

NOTICE: init_env_func()
NOTICE: generate_slave_list_func()
NOTICE: Your slaves are: ['pgdb02']

为备库建归档日志目录

ssh pgdb02 "mkdir -p /var/lib/pgsql/archive"

为cmd_standby创建辅助函数

psql -U postgres <$HOME/pitrtools-1.2/cmd_standby.sql

5. 主库打开归档模式

sed -i "
\$a \
archive_mode = 'on'\\
archive_command = '\$HOME/pitrtools-1.2/cmd_archiver -C \$HOME/pitrtools-1.2/cmd_archiver.ini -F %p \>\>\$HOME/pitrtools-1.2/cmd_archiver.\$(date \"+%%Y%%m%%d\").log'
/^archive_timeout/s/^/#/g
/^archive_mode\|^archive_command/d
" /var/lib/pgsql/data/postgresql.conf

pg_ctl restart -m fast

6. 备库配置standby

cat >$HOME/pitrtools-1.2/cmd_standby.ini <<EOF
[DEFAULT]
pgversion: 8.3
numarchives: 10
ssh: /usr/bin/ssh
rsync: /usr/bin/rsync
pg_standby: /usr/bin/pg_standby
pg_ctl: /usr/bin/pg_ctl
r_psql: /usr/bin/psql
port: 5432
master_public_ip: pgdb01
master_local_ip: 127.0.0.1
user: postgres
debug: off
ssh_timeout: 30
archivedir: /var/lib/pgsql/archive
pgdata: /var/lib/pgsql/data
postgresql_conf: $HOME/pitrtools-1.2/postgresql.conf
pg_hba_conf: $HOME/pitrtools-1.2/pg_hba.conf
notify_critical: echo critical.
notify_warning: echo warning.
notify_ok: echo ok.
action_failover: echo failover.
EOF

初始化创建备库目录

scp pgdb01:/var/lib/pgsql/data/postgresql.conf $HOME/pitrtools-1.2/
scp pgdb01:/var/lib/pgsql/data/pg_hba.conf $HOME/pitrtools-1.2/
sed -i '/^archive_/s/^/#/g' $HOME/pitrtools-1.2/postgresql.conf
rm -rf /var/lib/pgsql/data
$HOME/pitrtools-1.2/cmd_standby -C $HOME/pitrtools-1.2/cmd_standby.ini -I

7. 备份主库到备库

$HOME/pitrtools-1.2/cmd_standby -C $HOME/pitrtools-1.2/cmd_standby.ini -B

8. 启动备库

cp -p $HOME/pitrtools-1.2/{postgresql.conf,pg_hba.conf} /var/lib/pgsql/data/
$HOME/pitrtools-1.2/cmd_standby -C $HOME/pitrtools-1.2/cmd_standby.ini -S

至此备库配置完成

9. 测试主库是否连通

$ $HOME/pitrtools-1.2/cmd_standby -C $HOME/pitrtools-1.2/cmd_standby.ini -P
SUCCESS: Master returned: 1

10. 打开备库

$ $HOME/pitrtools-1.2/cmd_standby -C $HOME/pitrtools-1.2/cmd_standby.ini -F999
waiting for server to shut down... done
server stopped
server starting
failover.
NOTICE: Statistics are not replicated in warm standy mode.
HINT: Execute ANALYZE on your databases

11. 总结
同Walmgr相比, pitrtools用户必须手工创建一些目录和修改配置文件, 不够自动化, 不能保留主库备份, 程序也写得略显粗糙, 不够专业, 有不少有待完善的地方

外部链接:
Pitr Made Easy

-fin-

setting up postgresql warm standby

2009-10-13T17:59:00.002+08:00

setting up postgresql warm standby
配置Postgresql备库

其工作原理跟Oracle备库一样, Postgresql数据库也支持将所有操作记录到日志文件(write-ahead log, WAL), 支持定时归档日志文件, 传输到备库服务器上, 然后在备库服务器上应用归档日志, 连续进行基于时间点的恢复(Point-In-Time Recovery), 这种高可用功能被称为warm standby或log shipping.

测试环境:
主库服务器: pgdb01
备库服务器: pgdb02
数据库版本: Postgresql 8.3.7
操作系统版本: CentOS 5.3

1. 设置服务器之间ssh认证, 用于scp拷归档日志
pgdb01上生成公私密钥文件, 配置认证信息, 并将同样的密钥文件和配置信息复制到pgdb02

ssh-keygen -t dsa -N "" -f $HOME/.ssh/id_dsa
[ -f ~/.ssh/authorized_keys ] && sed -i '/'$(whoami)'@'$(hostname)'/d' ~/.ssh/authorized_keys
cat ~/.ssh/id_dsa.pub >>~/.ssh/authorized_keys
scp -pr $HOME/.ssh pgdb02:

测试一下

$ ssh pgdb02 "hostname -s"
pgdb02
$ ssh pgdb02 "ssh pgdb01 'hostname -s'"
pgdb01

2. 主库pgdb01开启日志归档/传输功能
备库pgdb02上创建归档日志存放目录:
mkdir -p /var/lib/pgsql/archlog/

编辑主库pgdb01配置文件

vi /var/lib/pgsql/data/postgresql.conf

增加如下配置项

archive_mode = on
archive_command = 'scp -C -o ConnectTimeout=30 "%p" pgdb02:/var/lib/pgsql/archlog/"%f"'
archive_timeout = 1800

archive_mode:on表示开启归档功能,off表示关闭. 重启后此参数才能生效
archive_command:归档命令, 调用scp拷贝归档文件至备库
archive_timeout:表示每隔多少秒强制归档一次

重启数据库, 使配置生效

pg_ctl restart -m fast

或用immediate参数停

pg_ctl restart -m immediate

3. 检查日志传输情况
调用pg_switch_xlog手工切换几次日志

$ psql -c "select pg_switch_xlog();"
 pg_switch_xlog
----------------
 2/21000088
(1 row)

如果当前日志中没有任何事务, pg_switch_xlog不会切换日志, 见9.23. System Administration Functions

查看pg_xlog目录

$ psql -c "select pg_current_xlog_location();"
 pg_current_xlog_location
--------------------------
 2/82000000
(1 row)

$ ls -lR /var/lib/pgsql/data/pg_xlog
/var/lib/pgsql/data/pg_xlog:
total 49216
-rw------- 1 postgres postgres 16777216 Oct 12 07:27 00000001000000020000007F
-rw------- 1 postgres postgres 16777216 Oct 12 07:31 000000010000000200000080
-rw------- 1 postgres postgres 16777216 Oct 12 07:32 000000010000000200000081
drwxr-xr-x 2 postgres postgres     4096 Oct 12 07:32 archive_status

/var/lib/pgsql/data/pg_xlog/archive_status:
total 0
-rw------- 1 postgres postgres 0 Oct 12 07:27 00000001000000020000007F.done
-rw------- 1 postgres postgres 0 Oct 12 07:31 000000010000000200000080.done
-rw------- 1 postgres postgres 0 Oct 12 07:32 000000010000000200000081.done

archive_status目录下.done文件表示已传输成功日志的编号, 由postgresql维护,自动删除

查看pgdb02, 归档日志已传到指定目录

$ ls -l /var/lib/pgsql/archlog/
total 49212
-rw------- 1 postgres postgres 16777216 Oct 12 07:27 00000001000000020000007F
-rw------- 1 postgres postgres 16777216 Oct 12 07:31 000000010000000200000080
-rw------- 1 postgres postgres 16777216 Oct 12 07:32 000000010000000200000081

4. 备份主库, 恢复到备库服务器上
备库pgdb02停止数据库实例

pg_ctl stop -m fast

删除data目录

rm -rf /var/lib/pgsql/data

pgdb01备份主库, 传输至备库

psql -c "checkpoint;select pg_start_backup('dbbackup');"
tar -C /var/lib/pgsql/ -zcf - data |ssh pgdb02 "tar -C /var/lib/pgsql/ -zxf -"
ssh pgdb02 'cd /var/lib/pgsql/data && rm -rf pg_log/* && find pg_xlog/ -type f -exec rm -f {} \;'
psql -c "select pg_stop_backup();"

$ psql -c "checkpoint;select pg_start_backup('dbbackup');"
 pg_start_backup
-----------------
 2/820000B0
(1 row)

$ tar -C /var/lib/pgsql/ -zcf - data |ssh pgdb02 "tar -C /var/lib/pgsql/ -zxf -"
$ ssh pgdb02 'cd /var/lib/pgsql/data && rm -rf pg_log/* && find pg_xlog/ -type f -exec rm -f {} \;'
$ psql -c "select pg_stop_backup();"
 pg_stop_backup
----------------
 2/82000118
(1 row)

备份产生的.backup文件记录了备份信息

$ ls -lR /var/lib/pgsql/data/pg_xlog/
/var/lib/pgsql/data/pg_xlog/:
total 65624
-rw------- 1 postgres postgres 16777216 Oct 12 07:56 000000010000000200000082
-rw------- 1 postgres postgres      244 Oct 12 07:56 000000010000000200000082.000000B0.backup
-rw------- 1 postgres postgres 16777216 Oct 12 07:27 000000010000000200000083
-rw------- 1 postgres postgres 16777216 Oct 12 07:32 000000010000000200000084
-rw------- 1 postgres postgres 16777216 Oct 12 07:31 000000010000000200000085
drwxr-xr-x 2 postgres postgres     4096 Oct 12 07:56 archive_status

/var/lib/pgsql/data/pg_xlog/archive_status:
total 0
-rw------- 1 postgres postgres 0 Oct 12 07:56 000000010000000200000082.000000B0.backup.done
-rw------- 1 postgres postgres 0 Oct 12 07:56 000000010000000200000082.done
$ cat /var/lib/pgsql/data/pg_xlog/000000010000000200000082.000000B0.backup
START WAL LOCATION: 2/820000B0 (file 000000010000000200000082)
STOP WAL LOCATION: 2/82000118 (file 000000010000000200000082)
CHECKPOINT LOCATION: 2/820000B0
START TIME: 2009-10-12 07:55:13 UTC
LABEL: dbbackup
STOP TIME: 2009-10-12 07:56:28 UTC

表示是从000000B0这个位置开始备份的, 解释见24.3.2. Making a Base Backup

5. 配置备库
备库上不需要归档, 所以修改备库配置文件, 关闭归档

sed -i '/^archive_/s/^/#/g' /var/lib/pgsql/data/postgresql.conf

安装postgresql contrib软件包, 使用其中的pg_standby辅助工具

sudo rpm -Uvh http://ftp9.us.postgresql.org/pub/mirrors/postgresql/binary/v8.3.7/linux/rpms/redhat/rhel-5-x86_64/postgresql-contrib-8.3.7-1PGDG.rhel5.x86_64.rpm

编辑恢复配置文件recovery.conf

cat >/var/lib/pgsql/data/recovery.conf <<'EOF'
restore_command = 'pg_standby -l -d -s 2 -t /tmp/pgsql.trigger.5432 /var/lib/pgsql/archlog %f %p %r 2>>pg_log/standby.log.$(date +%%Y%%m%%d)'
EOF

-l: 创建归档日志的符号链接, 而不是拷贝
-d: 打开调试信息
-s 2: 每2秒检查一次是否有新的归档日志
-t /tmp/pgsql.trigger.5432: 一旦发现存在触发器文件, 将终止恢复过程, 然后打开备库
各选项解释见F.23. pg_standby

启动备库
pg_ctl start

日志显示已恢复了所有归档日志, 等待着下一个日志

Trigger file : /tmp/pgsql.trigger.5432
Waiting for WAL file : 00000001.history
WAL file path : /var/lib/pgsql/archlog/00000001.history
Restoring to... : pg_xlog/RECOVERYHISTORY
Sleep interval : 2 seconds
Max wait interval : 0 forever
Command for restore : ln -s -f "/var/lib/pgsql/archlog/00000001.history" "pg_xlog/RECOVERYHISTORY"
Keep archive history : 000000000000000000000000 and later
running restore : OK
Trigger file : /tmp/pgsql.trigger.5432
Waiting for WAL file : 000000010000000200000082.000000B0.backup
WAL file path : /var/lib/pgsql/archlog/000000010000000200000082.000000B0.backup
Restoring to... : pg_xlog/RECOVERYHISTORY
Sleep interval : 2 seconds
Max wait interval : 0 forever
Command for restore : ln -s -f "/var/lib/pgsql/archlog/000000010000000200000082.000000B0.backup" "pg_xlog/RECOVERYHISTORY"
Keep archive history : 000000000000000000000000 and later
running restore : OK
Trigger file : /tmp/pgsql.trigger.5432
Waiting for WAL file : 000000010000000200000082
WAL file path : /var/lib/pgsql/archlog/000000010000000200000082
Restoring to... : pg_xlog/RECOVERYXLOG
Sleep interval : 2 seconds
Max wait interval : 0 forever
Command for restore : ln -s -f "/var/lib/pgsql/archlog/000000010000000200000082" "pg_xlog/RECOVERYXLOG"
Keep archive history : 000000000000000000000000 and later
running restore : OK

Trigger file : /tmp/pgsql.trigger.5432
Waiting for WAL file : 000000010000000200000083
WAL file path : /var/lib/pgsql/archlog/000000010000000200000083
Restoring to... : pg_xlog/RECOVERYXLOG
Sleep interval : 2 seconds
Max wait interval : 0 forever
Command for restore : ln -s -f "/var/lib/pgsql/archlog/000000010000000200000083" "pg_xlog/RECOVERYXLOG"
Keep archive history : 000000010000000200000082 and later
WAL file not present yet. Checking for trigger file...
WAL file not present yet. Checking for trigger file...
WAL file not present yet. Checking for trigger file...
...
...
...

$ ls -lR /var/lib/pgsql/data/pg_xlog/
/var/lib/pgsql/data/pg_xlog/:
total 8
drwxr-xr-x 2 postgres postgres 4096 Oct 12 07:55 archive_status
lrwxrwxrwx 1 postgres postgres   63 Oct 12 08:11 RECOVERYHISTORY -> /var/lib/pgsql/archlog/000000010000000200000082.000000B0.backup

/var/lib/pgsql/data/pg_xlog/archive_status:
total 0
[postgres@pgdb02.cloud101.bqa4.blurdev.com ~]$ ls -lR /var/lib/pgsql/archlog/
/var/lib/pgsql/archlog/:
total 65620
-rw------- 1 postgres postgres 16777216 Oct 12 07:27 00000001000000020000007F
-rw------- 1 postgres postgres 16777216 Oct 12 07:31 000000010000000200000080
-rw------- 1 postgres postgres 16777216 Oct 12 07:32 000000010000000200000081
-rw------- 1 postgres postgres 16777216 Oct 12 07:56 000000010000000200000082
-rw------- 1 postgres postgres      244 Oct 12 07:56 000000010000000200000082.000000B0.backup

pg_xlog目录下RECOVERYHISTORY文件不知有什么用
archlog目录下归档文件由postgresql维护, 成功恢复后, 会自动删除一些旧的

6. 打开备库
备库处在恢复状态时不能进行任何操作, 必须打开数据库后才能查询SQL

$ psql
psql: FATAL:  the database system is starting up

当pg_standby检测到触发器文件, 就会终止恢复过程, 打开数据库

touch /tmp/pgsql.trigger.5432

日志:

...
...
...
WAL file not present yet. Checking for trigger file...
WAL file not present yet. Checking for trigger file...
WAL file not present yet. Checking for trigger file...trigger file found

Trigger file : /tmp/pgsql.trigger.5432
Waiting for WAL file : 000000010000000200000085
WAL file path : /var/lib/pgsql/archlog/000000010000000200000085
Restoring to... : pg_xlog/RECOVERYXLOG
Sleep interval : 2 seconds
Max wait interval : 0 forever
Command for restore : ln -s -f "/var/lib/pgsql/archlog/000000010000000200000085" "pg_xlog/RECOVERYXLOG"
Keep archive history : 000000000000000000000000 and later
running restore : OK

Trigger file : /tmp/pgsql.trigger.5432
Waiting for WAL file : 00000002.history
WAL file path : /var/lib/pgsql/archlog/00000002.history
Restoring to... : pg_xlog/RECOVERYHISTORY
Sleep interval : 2 seconds
Max wait interval : 0 forever
Command for restore : ln -s -f "/var/lib/pgsql/archlog/00000002.history" "pg_xlog/RECOVERYHISTORY"
Keep archive history : 000000000000000000000000 and later
running restore : OK
Trigger file : /tmp/pgsql.trigger.5432
Waiting for WAL file : 00000001.history
WAL file path : /var/lib/pgsql/archlog/00000001.history
Restoring to... : pg_xlog/RECOVERYHISTORY
Sleep interval : 2 seconds
Max wait interval : 0 forever
Command for restore : ln -s -f "/var/lib/pgsql/archlog/00000001.history" "pg_xlog/RECOVERYHISTORY"
Keep archive history : 000000000000000000000000 and later
running restore : OK

恢复过程停止后, recovery.conf被自动改名为recovery.done

$ ls -lR /var/lib/pgsql/data/pg_xlog/
/var/lib/pgsql/data/pg_xlog/:
total 16412
lrwxrwxrwx 1 postgres postgres       47 Oct 12 09:28 000000020000000200000085 -> /var/lib/pgsql/archlog/000000010000000200000085
-rw------- 1 postgres postgres 16777216 Oct 12 09:28 000000020000000200000086
-rw------- 1 postgres postgres       74 Oct 12 09:28 00000002.history
drwxr-xr-x 2 postgres postgres     4096 Oct 12 09:28 archive_status

/var/lib/pgsql/data/pg_xlog/archive_status:
total 0
-rw------- 1 postgres postgres 0 Oct 12 09:28 00000002.history.ready
$ ls -lR /var/lib/pgsql/data/recovery.*
-rw-r--r-- 1 postgres postgres 142 Oct 12 08:11 /var/lib/pgsql/data/recovery.done

要注意的是, 一旦打开就无法再次进入恢复状态, 只能重建备库

外部链接:
Chapter 25. High Availability, Load Balancing, and Replication
Chapter 24. Backup and Restore
Simple HA with PostgreSQL Point-In-Time Recovery
F.23. pg_standby
Using pg_standby for high availability of Postgresql

7. 记录级别的日志传送(Record-based Log Shipping)
前面实现的是文件级别的日志传送, 以文件为单位进行恢复, 即使设置了定时强制归档, 恢复窗口也太大.
Postgresql支持记录级别的日志传送, 见24.4.4. Record-based Log Shipping, 但没提供相应的工具. 所幸的是Skype公司开发了一套SkyTools, 给广大数据库管理员带去了福音, 其中一款工具walmgr可以方便的创建/管理备库, 并实现了记录级别的日志传送.

8. 安装SkyTools
主备库服务器上安装SkyTools

rpm -Uvh http://yum.pgsqlrpms.org/8.3/redhat/rhel-5-x86_64/compat-postgresql-libs-4-1PGDG.rhel5.x86_64.rpm
yum -y install python-psycopg2.x86_64
rpm -Uvh http://yum.pgsqlrpms.org/8.3/redhat/rhel-5-x86_64/skytools-2.1.10-1.rhel5.x86_64.rpm
chmod +x /usr/bin/*.py

9. 配置walmgr
说明见walmgr(1)

主库:

mkdir -p /var/lib/pgsql/walmgr/logs
cat >/var/lib/pgsql/walmgr/master.ini <<'EOF'
[wal-master]
job_name             = pgdb01.cloud101_walgmr_master
logfile              = /var/lib/pgsql/walmgr/logs/master.log
pidfile              = /var/lib/pgsql/walmgr/logs/master.pid
master_db            = dbname=template1
master_data          = /var/lib/pgsql/data
master_config        = /var/lib/pgsql/data/postgresql.conf
slave                = pgdb02:/var/lib/pgsql/walmgr/shipping
slave_config         = /var/lib/pgsql/walmgr/slave.ini
completed_wals       = %(slave)s/logs.complete
partial_wals         = %(slave)s/logs.partial
full_backup          = %(slave)s/data.master
config_backup        = %(slave)s/config.backup
loop_delay           = 10.0
use_xlog_functions   = 1
compression          = 1
EOF

备库:

mkdir -p /var/lib/pgsql/walmgr/{logs,shipping/{logs.{complete,partial},data.master,config.backup}}
cat >/var/lib/pgsql/walmgr/slave.ini <<'EOF'
[wal-slave]
job_name             = pgdb02.cloud101_walgmr_slave
logfile              = /var/lib/pgsql/walmgr/logs/slave.log
slave_data           = /var/lib/pgsql/data
slave_stop_cmd       = sudo /etc/init.d/postgresql stop
slave_start_cmd      = sudo /etc/init.d/postgresql start
slave_config_dir     = /var/lib/pgsql/data
slave                = /var/lib/pgsql/walmgr/shipping
completed_wals       = %(slave)s/logs.complete
partial_wals         = %(slave)s/logs.partial
full_backup          = %(slave)s/data.master
config_backup        = %(slave)s/config.backup
keep_backups         = 5
EOF

10. 配置主库

$ walmgr.py /var/lib/pgsql/walmgr/master.ini setup
2009-10-12 10:09:58,499 29592 INFO Configuring WAL archiving
2009-10-12 10:09:58,502 29592 WARNING database must be restarted to enable archiving
2009-10-12 10:09:58,506 29592 INFO Sending SIGHUP to postmaster
2009-10-12 10:09:59,222 29592 INFO Done

运行完setup,配置文件增加了如下2条配置项

archive_mode = 'on'
archive_command = '/usr/bin/walmgr.py /var/lib/pgsql/walmgr/master.ini xarchive %p %f'

重启使修改生效

pg_ctl restart -m fast

11. 备份主库

$ walmgr.py /var/lib/pgsql/walmgr/master.ini backup
2009-10-12 10:27:05,706 18300 INFO Backup lock obtained.
2009-10-12 10:27:05,706 18300 INFO got SystemExit(0), exiting
2009-10-12 10:27:05,741 29813 INFO Execute SQL: select pg_start_backup('FullBackup'); [dbname=template1]
2009-10-12 10:27:07,696 29813 INFO Checking tablespaces
2009-10-12 10:27:08,776 29813 INFO Backup conf files from /var/lib/pgsql/data
2009-10-12 10:27:09,268 18323 INFO First useful WAL file is: 00000001000000020000008B
2009-10-12 10:27:09,744 18329 INFO Backup lock released.
2009-10-12 10:27:09,785 29813 INFO Full backup successful

数据库备份到了备库服务器指定目录上
数据文件放在/var/lib/pgsql/walmgr/shipping/data.master
配置文件放在/var/lib/pgsql/walmgr/shipping/config.backup

日志:

2009-10-12 10:27:05,741 29813 INFO Execute SQL: select pg_start_backup('FullBackup'); [dbname=template1]
2009-10-12 10:27:07,696 29813 INFO Checking tablespaces
2009-10-12 10:27:08,776 29813 INFO Backup conf files from /var/lib/pgsql/data
2009-10-12 10:27:09,785 29813 INFO Full backup successful
2009-10-12 10:27:09,852 29825 INFO {count: 1, duration: 0.387887001038}
2009-10-12 10:27:10,724 29829 INFO {count: 1, duration: 0.725953102112}
2009-10-12 10:27:11,272 29832 INFO {count: 1, duration: 0.396477937698}
2009-10-12 10:27:12,056 29835 INFO {count: 1, duration: 0.628087997437}

12. 启用备库

$ walmgr.py /var/lib/pgsql/walmgr/slave.ini restore data.master
2009-10-12 10:28:32,363 18411 INFO Stopping postmaster: sudo /etc/init.d/postgresql stop
Stopping postgresql service:                               [  OK  ]
2009-10-12 10:28:35,535 18411 INFO Move /var/lib/pgsql/data to /var/lib/pgsql/data.2
2009-10-12 10:28:35,536 18411 INFO Copy /var/lib/pgsql/walmgr/shipping/data.master to /var/lib/pgsql/data
2009-10-12 10:28:36,001 18411 INFO Write /var/lib/pgsql/data/recovery.conf
2009-10-12 10:28:36,003 18411 INFO Restoring configuration files
2009-10-12 10:28:36,007 18411 INFO Starting postmaster: sudo /etc/init.d/postgresql start
Starting postgresql service:                               [  OK  ]

这样备库就建好了

日志:

2009-10-12 10:28:32,363 18411 INFO Stopping postmaster: sudo /etc/init.d/postgresql stop
2009-10-12 10:28:35,535 18411 INFO Move /var/lib/pgsql/data to /var/lib/pgsql/data.2
2009-10-12 10:28:35,536 18411 INFO Copy /var/lib/pgsql/walmgr/shipping/data.master to /var/lib/pgsql/data
2009-10-12 10:28:36,001 18411 INFO Write /var/lib/pgsql/data/recovery.conf
2009-10-12 10:28:36,003 18411 INFO Restoring configuration files
2009-10-12 10:28:36,007 18411 INFO Starting postmaster: sudo /etc/init.d/postgresql start
2009-10-12 10:28:37,094 18511 INFO 00000001.history: not found, ignoring
2009-10-12 10:28:37,094 18511 INFO got SystemExit(1), exiting
2009-10-12 10:28:37,233 18512 INFO 00000001000000020000008B.00000020.backup: Found
2009-10-12 10:28:37,243 18512 INFO {count: 1}
2009-10-12 10:28:37,387 18514 INFO 00000001000000020000008B: Found
2009-10-12 10:28:37,489 18514 INFO {count: 1}

13. 手工同步
在主库上运行

$ walmgr.py /var/lib/pgsql/walmgr/master.ini sync
2009-10-12 10:30:29,591 29862 INFO sent to slave: 00000001000000020000008B @ 0 +16777216

14. 打开备库

$ walmgr.py /var/lib/pgsql/walmgr/slave.ini boot
2009-10-12 10:31:38,781 18534 INFO Stopping recovery mode

日志:

2009-10-12 10:31:38,781 18534 INFO Stopping recovery mode
2009-10-12 10:31:39,485 18516 INFO 00000001000000020000008C: not found, stopping
2009-10-12 10:31:39,485 18516 INFO got SystemExit(1), exiting
2009-10-12 10:31:39,628 18536 INFO 00000001000000020000008B: Found
2009-10-12 10:31:39,737 18536 INFO {count: 1}
2009-10-12 10:31:39,881 18538 INFO 00000002.history: not found, ignoring
2009-10-12 10:31:39,882 18538 INFO got SystemExit(1), exiting
2009-10-12 10:31:40,024 18539 INFO 00000001.history: not found, ignoring
2009-10-12 10:31:40,024 18539 INFO got SystemExit(1), exiting

15. 恢复备库
备库打开后无法切换回恢复状态, 必须重新从备份重建. 操作同步骤12

walmgr.py /var/lib/pgsql/walmgr/slave.ini restore data.master

16. 自动定时同步
主库上运行

nohup walmgr.py /var/lib/pgsql/walmgr/master.ini syncdaemon >>/var/lib/pgsql/walmgr/logs/syncdaemon.log 2>&1 </dev/null &

按loop_delay指定的时间间隔同步

17. 切换主备库
假设pgdb01是主库, pgdb02是备库, 切换主备库角色

1) 按新的主备库角色编辑配置文件
pgdb02:

mkdir -p /var/lib/pgsql/walmgr/logs
cat >/var/lib/pgsql/walmgr/master.ini <<'EOF'
[wal-master]
job_name             = pgdb02.cloud101_walgmr_master
logfile              = /var/lib/pgsql/walmgr/logs/master.log
pidfile              = /var/lib/pgsql/walmgr/logs/master.pid
master_db            = dbname=template1
master_data          = /var/lib/pgsql/data
master_config        = /var/lib/pgsql/data/postgresql.conf
slave                = pgdb01:/var/lib/pgsql/walmgr/shipping
slave_config         = /var/lib/pgsql/walmgr/slave.ini
completed_wals       = %(slave)s/logs.complete
partial_wals         = %(slave)s/logs.partial
full_backup          = %(slave)s/data.master
config_backup        = %(slave)s/config.backup
loop_delay           = 10.0
use_xlog_functions   = 1
compression          = 1
EOF

pgdb01:

mkdir -p /var/lib/pgsql/walmgr/{logs,shipping/{logs.{complete,partial},data.master,config.backup}}
cat >/var/lib/pgsql/walmgr/slave.ini <<'EOF'
[wal-slave]
job_name             = pgdb01.cloud101_walgmr_slave
logfile              = /var/lib/pgsql/walmgr/logs/slave.log
slave_data           = /var/lib/pgsql/data
slave_stop_cmd       = sudo /etc/init.d/postgresql stop
slave_start_cmd      = sudo /etc/init.d/postgresql start
slave_config_dir     = /var/lib/pgsql/data
slave                = /var/lib/pgsql/walmgr/shipping
completed_wals       = %(slave)s/logs.complete
partial_wals         = %(slave)s/logs.partial
full_backup          = %(slave)s/data.master
config_backup        = %(slave)s/config.backup
keep_backups         = 5
EOF

2) 停止应用程序, 将应用程序的数据库配置指向pgdb02

3) pgdb01强制同步一次

walmgr.py /var/lib/pgsql/walmgr/master.ini sync

4) 停止pgdb01的后台同步进程

kill $(cat /var/lib/pgsql/walmgr/logs/master.pid)

5) 打开pgdb02

walmgr.py /var/lib/pgsql/walmgr/slave.ini boot

6) 配置pgdb02, 启用归档

walmgr.py /var/lib/pgsql/walmgr/master.ini setup
pg_ctl restart -m fast

7) 现在pgdb02是主库, 启动应用程序, 连接pgdb02

8) 备份pgdb02

walmgr.py /var/lib/pgsql/walmgr/master.ini backup

9) 恢复到pgdb01

walmgr.py /var/lib/pgsql/walmgr/slave.ini restore data.master

10) pgdb02启用后台自动定时同步

nohup walmgr.py /var/lib/pgsql/walmgr/master.ini syncdaemon >>/var/lib/pgsql/walmgr/logs/syncdaemon.log 2>&1 </dev/null &

11) 完成

外部链接:
walmgr
walmgr(1)
Testing PostgreSQL replication solutions: Log shipping with walmgr
Core Services 2.0 skytools setup

18. 其它复制/集群/高可用方案

介绍:
Replication, Clustering, and Connection Pooling
PostgreSQL Replication Solutions - Presentation
PostgreSQL Replication
The Four Horsemen of Replication 1 2 3
Getting Smart about the New World of PostgreSQL Replication

本人认为比较简单好用的几种:

Warm Standby: 基于日志的异步复制, 复制时备库不可用
Hot Standby: 备库可读. postgresql当前版本不支持此功能, 8.5版本开始支持
Mammoth Replicator: Postgresql的修改版, 支持主从异步复制, , 使用上有很多限制, 比如只支持复制一个数据库(schema)
Tungsten Replicator: 支持主从复制, 基于日志分析, 性能很差, 目前只支持Mysql,Oracle复制, 貌似很有前途, 值得一用
链接:
Continuent Tungsten Replicator Guide
The Scale-Out Blog
RubyRep: Ruby脚本编写的, 支持Mysql和Postgresql, 跟据主键或唯一索引比较2个数据库中表的不同, 进行同步. 可以实现主从或主主复制. 估计性能不行
例子:Yet Another PostgreSQL Replication Tool - RubyRep

19. Postgresql数据库下载地址
http://www.postgresql.org/ftp/
https://projects.commandprompt.com/public/pgcore/wiki/direct_download

-fin-

sort file by length of lines

2009-10-12T14:33:00.002+08:00

sort file by length of lines
按每行长度对文件内容排序

比如文件包含一些人名, 想按名称长度进行排序

cat >myfile <<EOF
Andre Anichanov
Andrew Mogrelia
Antonio Vivaldi
Antoni Wit
Capella Istropolitana
Christian Kohn
Cologne Chamber Orchestra
Daniel Rothert
Daniel Taylor
Einar Steen-Nokleberg
Elena Papandreou
Eteri Andjaparidze
EOF

awk得到每行的长度, 按长度排序后显示输出

awk '{print length,$0}' myfile|sort -rn|cut -d" " -f2-

或者将管道写在print后
awk '{print length,$0|"sort -rn|cut -d\" \" -f2-"}' myfile

运行结果

Cologne Chamber Orchestra
Einar Steen-Nokleberg
Capella Istropolitana
Eteri Andjaparidze
Elena Papandreou
Antonio Vivaldi
Andrew Mogrelia
Andre Anichanov
Daniel Rothert
Christian Kohn
Daniel Taylor
Antoni Wit

-fin-

introducing maatkit - parallel dump

2009-09-23T20:02:00.013+08:00

introducing maatkit - parallel dump
maatkit介绍 - 并行导出

Maatkit是一组为MySQL提供的命令行工具集, 是由Percona公司开发的开源软件
该公司同时还开发了XtraDB存储引擎, XtraBackup热备工具, MySQL增强补丁版本Percona

包括:

mk-archiver 将表数据清除或归档到另外的表或文件
mk-audit 分析MySQL的配置,概要,操作, 生成报表
mk-checksum-filter mk-table-checksum的过滤器
mk-deadlock-logger 记录InnoDB的死锁信息
mk-duplicate-key-checker 查找重复或冗余的外键和索引
mk-fifo-split 将一个文件拆分为多个部分, 输出到FIFO管道(有用吗?)
mk-find 按指定规则查找表名, 然后执行操作
mk-heartbeat 监视数据库之间复制的延迟
mk-log-player 拆分并重演慢速查询日志
mk-parallel-dump 多线程导出
mk-parallel-restore 多线程导入
mk-profile-compact 压缩mk-query-profiler输出
mk-query-digest 分析日志
mk-query-profiler 查询性能分析工具
mk-show-grants 显示用户权限
mk-slave-delay 实现备库与主库之间一定的延时
mk-slave-find 查找/显示出备库的树型层次结构
mk-slave-move 在层次结构中移动备库(什么玩意?)
mk-slave-prefetch 在备库上运行SELECT查询语句, 使数据预读取到内存中
mk-slave-restart 监测备库发生的错误并重启
mk-table-checksum 快速检测两个表的数据是否相同. 可以用来检测备库和主库的数据一致性
mk-table-sync 发现并修复不同服务器上的两个表之间的数据差异
mk-upgrade 比较2个数据库中语句的运行结果
mk-visual-explain 以树形显示执行计划

1. 安装
从http://maatkit.googlecode.com/处下载RPM包进行安装

yum -y install perl-TermReadKey.x86_64
rpm -Uvh http://maatkit.googlecode.com/files/maatkit-4623-1.noarch.rpm

依赖以下安装包

# rpm -q --requires maatkit
/usr/bin/env
perl(DBD::mysql) >= 1.0
perl(DBI)
perl(DBI) >= 1.13
perl(Data::Dumper)
perl(Digest::MD5)
perl(English)
perl(Exporter)
perl(File::Basename)
perl(File::Find)
perl(File::Spec)
perl(File::Temp)
perl(Getopt::Long)
perl(IO::File)
perl(List::Util)
perl(POSIX)
perl(Socket)
perl(Term::ReadKey) >= 2.10
perl(Time::HiRes)
perl(Time::Local)
perl(constant)
perl(sigtrap)
perl(strict)
perl(warnings)
rpmlib(CompressedFileNames) <= 3.0.4-1
rpmlib(PayloadFilesHavePrefix) <= 4.0-1

2. 并行导出mk-parallel-dump和导入mk-parallel-restore
使用方法见:
mk-parallel-dump - Dump sets of MySQL tables in parallel.
mk-parallel-restore - Load files into MySQL in parallel.

1) 生成测试表
生成一个3百万行记录的测试表

mysql --socket=/var/lib/mysql/data_3306/mysql.sock

set autocommit=0;
drop database if exists dbtest;
create database dbtest;
use dbtest;
drop table if exists t1;
create table t1 (
  id int(9) not null auto_increment,
  name varchar(20) not null,
  age int(3) not null,
  notes varchar(100),
  primary key (id),
  index ind_t1_name (name)
);
truncate table t1;
insert into t1 (name, age, notes)
select conv(floor(rand() * 99999999999999), 10, 36), floor(1+rand()*(100-1)), md5(rand())
  from information_schema.COLUMNS a
       , information_schema.COLUMNS b
       , information_schema.COLUMNS c
 limit 3000000;
commit;

大小300多M

mysql> insert into t1 (name, age, notes)
    -> select conv(floor(rand() * 99999999999999), 10, 36), floor(1+rand()*(100-1)), md5(rand())
    ->   from information_schema.COLUMNS a
    ->        , information_schema.COLUMNS b
    ->        , information_schema.COLUMNS c
    ->  limit 3000000;
Query OK, 3000000 rows affected (3 min 30.61 sec)
Records: 3000000  Duplicates: 0  Warnings: 0

mysql> commit;
Query OK, 0 rows affected (0.07 sec)
mysql> show table status like 't1';
+------+--------+---------+------------+---------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+-------------+------------+-----------------+----------+----------------+----------------------+
| Name | Engine | Version | Row_format | Rows    | Avg_row_length | Data_length | Max_data_length | Index_length | Data_free | Auto_increment | Create_time         | Update_time | Check_time | Collation       | Checksum | Create_options | Comment              |
+------+--------+---------+------------+---------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+-------------+------------+-----------------+----------+----------------+----------------------+
| t1   | InnoDB |      10 | Compact    | 3000249 |             76 |   228294656 |               0 |     86654976 |         0 |        6000000 | 2009-09-23 05:26:34 | NULL        | NULL       | utf8_general_ci |     NULL |                | InnoDB free: 4096 kB |
+------+--------+---------+------------+---------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+-------------+------------+-----------------+----------+----------------+----------------------+
1 row in set (0.08 sec)

mysql> system ls -l /var/lib/mysql/data_3306/dbtest
total 319820
-rw-rw---- 1 mysql mysql        61 Sep 23 04:17 db.opt
-rw-rw---- 1 mysql mysql      8646 Sep 23 05:26 t1.frm
-rw-rw---- 1 mysql mysql 327155712 Sep 23 05:30 t1.ibd
mysql>

2) mk-parallel-dump和mysqldump分别导出该表作比较
mysqldump导出表

# mkdir -p $HOME/backup
# cd $HOME/backup && rm -rf *
# time mysqldump --socket=/var/lib/mysql/data_3306/mysql.sock --opt dbtest t1 >dbback-dbtest-t1.sql

real    0m11.316s
user    0m2.348s
sys     0m0.472s
# ls -l dbback-dbtest-t1.sql
-rw-r--r-- 1 root root 179090589 Sep 23 05:31 dbback-dbtest-t1.sql

mk-parallel-dump导出表, 导出文件放在目录$HOME/backup/pdump1下, 不压缩, 不记录binlog的位置

# time mk-parallel-dump --socket=/var/lib/mysql/data_3306/mysql.sock --base-dir=$HOME/backup/pdump1 --no-gzip --nobin-log-position --tables="dbtest.t1"
     default:              1 tables,     1 chunks,     1 successes,  0 failures,  11.40 wall-clock time,  11.31 dump time

real    0m11.608s
user    0m2.176s
sys     0m0.556s
# find pdump1 -ls
7162198    4 drwxr-xr-x   3 root     root         4096 Sep 23 05:33 pdump1
7162199    4 drwxr-xr-x   3 root     root         4096 Sep 23 05:33 pdump1/default
7162200    4 drwxr-xr-x   2 root     root         4096 Sep 23 05:33 pdump1/default/dbtest
7162201 175076 -rw-r--r--   1 root     root     179096039 Sep 23 05:33 pdump1/default/dbtest/t1.000000.sql

mk-parallel-dump导出表, 增加每一百万条(近似值)导出一个文件

# time mk-parallel-dump --socket=/var/lib/mysql/data_3306/mysql.sock --base-dir=$HOME/backup/pdump2 --no-gzip --nobin-log-position --tables="dbtest.t1" --chunk-size=1000000
     default:              1 tables,     4 chunks,     4 successes,  0 failures,  10.36 wall-clock time,  15.92 dump time

real    0m10.509s
user    0m2.580s
sys     0m0.560s
# find pdump2 -ls
7162202    4 drwxr-xr-x   3 root     root         4096 Sep 23 05:33 pdump2
7162203    4 drwxr-xr-x   3 root     root         4096 Sep 23 05:33 pdump2/default
7162204    4 drwxr-xr-x   2 root     root         4096 Sep 23 05:33 pdump2/default/dbtest
7162205    4 -rw-r--r--   1 root     root          101 Sep 23 05:33 pdump2/default/dbtest/t1.chunks
7162208 58544 -rw-r--r--   1 root     root     59879876 Sep 23 05:33 pdump2/default/dbtest/t1.000001.sql
7162206   16 -rw-r--r--   1 root     root        16365 Sep 23 05:33 pdump2/default/dbtest/t1.000003.sql
7162209 58000 -rw-r--r--   1 root     root     59324125 Sep 23 05:33 pdump2/default/dbtest/t1.000000.sql
7162207 58544 -rw-r--r--   1 root     root     59880064 Sep 23 05:33 pdump2/default/dbtest/t1.000002.sql
# cat pdump2/default/dbtest/t1.chunks
`id` < 1999835
`id` >= 1999835 AND `id` < 3999669
`id` >= 3999669 AND `id` < 5999503
`id` >= 5999503

.chunks文件记录了分块规则

mk-parallel-dump导出表, 增加启动4个线程同时导出(不指定则默认为2个线程)

# time mk-parallel-dump --socket=/var/lib/mysql/data_3306/mysql.sock --base-dir=$HOME/backup/pdump3 --no-gzip --nobin-log-position --tables="dbtest.t1" --chunk-size=1000000 --threads=4
     default:              1 tables,     4 chunks,     4 successes,  0 failures,   9.37 wall-clock time,  25.29 dump time

real    0m9.529s
user    0m2.572s
sys     0m0.516s
# find pdump3 -ls
7359077    4 drwxr-xr-x   3 root     root         4096 Sep 23 05:34 pdump3
7359078    4 drwxr-xr-x   3 root     root         4096 Sep 23 05:34 pdump3/default
7359079    4 drwxr-xr-x   2 root     root         4096 Sep 23 05:34 pdump3/default/dbtest
7359080    4 -rw-r--r--   1 root     root          101 Sep 23 05:34 pdump3/default/dbtest/t1.chunks
7359084 58544 -rw-r--r--   1 root     root     59879876 Sep 23 05:34 pdump3/default/dbtest/t1.000001.sql
7359081   16 -rw-r--r--   1 root     root        16365 Sep 23 05:34 pdump3/default/dbtest/t1.000003.sql
7359083 58000 -rw-r--r--   1 root     root     59324125 Sep 23 05:34 pdump3/default/dbtest/t1.000000.sql
7359082 58544 -rw-r--r--   1 root     root     59880064 Sep 23 05:34 pdump3/default/dbtest/t1.000002.sql

导出速度差不多, 无显著差异, 因为这是个单CPU的系统, 只导出一个表, 分块和多线程可能还会带来额外的开销.
如果是多核多CPU系统导出多个表, mk-parallel-dump应该会更快些.

3) 比较mk-parallel-dump和mysqldump导入
mysql导入

# time mysql --socket=/var/lib/mysql/data_3306/mysql.sock dbtest <dbback-dbtest-t1.sql

real    3m16.760s
user    0m1.672s
sys     0m0.156s

mk-parallel-restore导入

# time mk-parallel-restore --socket=/var/lib/mysql/data_3306/mysql.sock $HOME/backup/pdump1
    1 tables,     1 files,     1 successes,  0 failures, 199.75 wall-clock time, 199.75 load time

real    3m19.910s
user    0m0.232s
sys     0m0.136s

mk-parallel-restore导入多个文件

# mysql --socket=/var/lib/mysql/data_3306/mysql.sock -e "drop table dbtest.t1;"
# time mk-parallel-restore --socket=/var/lib/mysql/data_3306/mysql.sock $HOME/backup/pdump2
    1 tables,     4 files,     1 successes,  0 failures, 196.55 wall-clock time, 196.54 load time

real    3m16.653s
user    0m0.268s
sys     0m0.148s

mk-parallel-restore导入多个文件, 启4个线程

# mysql --socket=/var/lib/mysql/data_3306/mysql.sock -e "drop table dbtest.t1;"
# time mk-parallel-restore --socket=/var/lib/mysql/data_3306/mysql.sock $HOME/backup/pdump3 --threads=4
    1 tables,     4 files,     1 successes,  0 failures, 194.19 wall-clock time, 194.19 load time

real    3m14.606s
user    0m0.204s
sys     0m0.164s

速度也都差不多

总体感觉很一般啊, 以后再试试其它工具. 关于数据库复制, 日志/语句分析等工具可能还比较有用.

外部链接:
Tools for MySQL - Maatkit makes MySQL - easier to manage.
maatkit - A toolkit that provides advanced functionality for MySQL
mysql-parallel-dump test

-fin-

DML Error Logging

2009-09-21T19:12:00.002+08:00

DML Error Logging

提问:一个问题，在存储过程中，有对20万条记录进行修改的一个UPDATE,当中间某个表由于字段长度问题而导致修改失败，在异常处理中，如何定位是哪条记录问题导致失败？

这个无法显示哪行出现超出？你能定位到哪行??

SQL> CREATE OR REPLACE PROCEDURE test111 AS
  2 
  3    v_sqlcode number;
  4    v_sqlerrm varchar2(100);
  5  BEGIN
  6 
  7 
  8  update test set aa=aa||'string' ;
  9      commit;
 10  EXCEPTION
 11      when others then
 12      v_sqlcode:=Sqlcode;
 13      v_sqlerrm:=Sqlerrm;
 14       rollback;
 15  DBMS_OUTPUT.PUT_LINE('AAA' || SQLCODE || SQLERRM || 'START');
 16 
 17  END;
 18  /
 
Procedure created.
 
SQL> set serveroutput on
SQL> exec test111;
AAA-12899ORA-12899: value too large for column "HR"."TEST"."AA" (actual: 11,
maximum: 10)START

回答: 数据操纵语言错误记录(DML Error Logging)能够满足您的要求

运行SQL语句, 如果只有一条记录出错, 也会立刻终止语句运行, 回滚事务, 导致对数据的修改全部失败, 尤其是当用一条语句批量处理大量记录时这个问题更加突出
10gR2增加了DML错误记录功能, 在运行SQL语句发生某些异常时, 不会中断整个事务, 而是自动将错误信息记录到另一个指定的表, 然后继续处理

1. 举例
创建测试表

set serveroutput on size unlimited
set pages 50000 line 130
drop table t purge;
drop table err$_t purge;
create table t(a number(1) primary key, b char);

然后需要建一个记录错误信息的表, 用Oracle提供的DBMS_ERRLOG包自动创建或手工创建

exec dbms_errlog.create_error_log('t');

缺省名称是ERR$_加原表名的前25个字符

SQL> select * from tab;

TNAME                          TABTYPE  CLUSTERID
------------------------------ ------- ----------
T                              TABLE
ERR$_T                         TABLE

SQL> desc ERR$_T
 Name                                                                    Null?    Type
 ----------------------------------------------------------------------- -------- ------------------------------------------------
 ORA_ERR_NUMBER$                                                                  NUMBER
 ORA_ERR_MESG$                                                                    VARCHAR2(2000)
 ORA_ERR_ROWID$                                                                   ROWID
 ORA_ERR_OPTYP$                                                                   VARCHAR2(2)
 ORA_ERR_TAG$                                                                     VARCHAR2(2000)
 A                                                                                VARCHAR2(4000)
 B                                                                                VARCHAR2(4000)

SQL>

ORA_ERR_NUMBER$ 错误编号
ORA_ERR_MESG$ 错误信息
ORA_ERR_ROWID$ 出错行的rowid(只对update和delete)
ORA_ERR_OPTYP$ 错误类型 I:插入 U:更新 D:删除
ORA_ERR_TAG$ 由用户定义的标签
(如果采用手工创建, 必须包括上述字段)
后两个字段与原表对应, 数据类型为varchar2(4000), 用于存储出错的记录(数据类型转换见Table 15-2 Error Logging Table Column Data Types)

用原始方式插入测试数据

SQL> insert into t (a) select level from dual connect by level <= 12;
insert into t (a) select level from dual connect by level <= 12
                               *
ERROR at line 1:
ORA-01438: value larger than specified precision allowed for this column

字段a只有一位数字, 不能超过9, 插入10导致出错了

增加LOG ERRORS子句后

SQL> insert into t (a) select level from dual connect by level <= 12 log errors reject limit unlimited;

9 rows created.

SQL> select * from t;

         A B
---------- -
         1
         2
         3
         4
         5
         6
         7
         8
         9

9 rows selected.

SQL> col ORA_ERR_MESG$ for a50
SQL> col ORA_ERR_TAG$ for a10
SQL> col ORA_ERR_ROWID$ for a10
SQL> col A for a10
SQL> col b for a10
SQL> select * from err$_t;

ORA_ERR_NUMBER$ ORA_ERR_MESG$                                      ORA_ERR_RO OR ORA_ERR_TA A          B
--------------- -------------------------------------------------- ---------- -- ---------- ---------- ----------
           1438 ORA-01438: value larger than specified precision a            I             10
                llowed for this column

           1438 ORA-01438: value larger than specified precision a            I             11
                llowed for this column

           1438 ORA-01438: value larger than specified precision a            I             12
                llowed for this column

SQL语句运行成功并且将错误记录到了err$_t

2. 语法
Error logging的语法是:
LOG ERRORS [INTO [schema.]table]
[ (simple_expression) ]
[ REJECT LIMIT {integer|UNLIMITED} ]

INTO子句可选, 缺省表名是err$_原表名的前25个字符
simple_expression可以是一个由表达式构成字符串, 作为为标签插入到字段ORA_ERR_TAG$
REJECT LIMIT表示最多允许记录多少个错误, 超过这个范围就抛出异常. 如果是0, 表示不记录错误(见下)

3. REJECT LIMIT 子句
在10.2.0.4测试结果和10gR2文档说的有些出入

缺省情况也记录错误, 只记录一条错误

SQL> truncate table t;

Table truncated.

SQL> truncate table err$_t;

Table truncated.

SQL> insert into t (a) select level from dual connect by level <= 13 log errors;
insert into t (a) select level from dual connect by level <= 13 log errors
                               *
ERROR at line 1:
ORA-01438: value larger than specified precision allowed for this column


SQL> select * from err$_t;

ORA_ERR_NUMBER$ ORA_ERR_MESG$                                                          ORA_ERR_RO OR ORA_ERR_TA A
--------------- ---------------------------------------------------------------------- ---------- -- ---------- ----------
B
----------
           1438 ORA-01438: value larger than specified precision allowed for this colu            I             10
                mn



SQL>

如果指定了数量n, 会记录n+1条

SQL> truncate table t;

Table truncated.

SQL> truncate table err$_t;

Table truncated.

SQL> insert into t (a) select level from dual connect by level <= 13 log errors reject limit 2;
insert into t (a) select level from dual connect by level <= 13 log errors reject limit 2
                               *
ERROR at line 1:
ORA-01438: value larger than specified precision allowed for this column


SQL> select * from err$_t;

ORA_ERR_NUMBER$ ORA_ERR_MESG$                                                          ORA_ERR_RO OR ORA_ERR_TA A
--------------- ---------------------------------------------------------------------- ---------- -- ---------- ----------
B
----------
           1438 ORA-01438: value larger than specified precision allowed for this colu            I             10
                mn


           1438 ORA-01438: value larger than specified precision allowed for this colu            I             11
                mn


           1438 ORA-01438: value larger than specified precision allowed for this colu            I             12
                mn



SQL>

11gR1文档似乎修正了这个错误:
This subclause indicates the maximum number of errors that can be encountered before the INSERT statement terminates and rolls back. You can also specify UNLIMITED. The default reject limit is zero, which means that upon encountering the first error, the error is logged and the statement rolls back. For parallel DML operations, the reject limit is applied to each parallel server.

11gR2文档:
If REJECT LIMIT X had been specified, the statement would have failed with the error message of error X=1. The error message can be different for different reject limits. In the case of a failing statement, only the DML statement is rolled back, not the insertion into the DML error logging table. The error logging table will contain X+1 rows.

4. 错误记录表
错误记录表不会自动清除, 以自治事务运行. 如超过REJECT LIMIT限制, DML语句回滚, 错误记录表不回滚
错误记录表所属的用户和运行DML语句的用户可以不相同, 运行语句的用户对错误记录表必须具有插入权限

5. 使用限制
对以下情况记录错误:

列值太大
违反非空,唯一,引用(referential),或检查(check)约束条件
由触发器抛出的异常
数据类型转换错误
分区映射错误
某些merge操作错误(如 ORA-30926: Unable to get a stable set of rows for MERGE operation.)

下述情况不记录错误:

违反了延迟的(deferred)约束条件
空间不够
直接路径插入操作(insert或merge)抛出的唯一约束或唯一索引错误
更新操作(update或merge)抛出的唯一约束或唯一索引错误

外部链接:
DML Error Logging
Error Logging and Handling Mechanisms
Inserting Data with DML Error Logging
38 DBMS_ERRLOG

Faster Batch Processing By Mark Rittman
10gR2 New Feature: DML Error Logging
DML Error Logging in Oracle 10g Database Release 2
Oracle DBMS_ERRLOG
Oracle DML Error Logging
dml error logging in oracle 10g release 2

-fin-

no login prompt in vm console

2009-09-03T18:28:00.000+08:00

no login prompt in vm console
虚拟机终端没有登录提示

用xm console命令登录Xen创建的虚拟机, 屏幕上只显示启动时的信息, 没有登录提示, 按多次回车也有没反应, 如下:

Mounting other filesystems:  [  OK  ]
Starting sshd: [  OK  ]
Starting crond: [  OK  ]
Starting anacron: [  OK  ]
Starting atd: [  OK  ]
Starting Avahi daemon... [  OK  ]
Starting HAL daemon: [  OK  ]
Starting puppet: [  OK  ]
Starting smartd: [  OK  ]
INIT: version 2.86 reloading

按文档Console handling指示
需要在/etc/inittab增加一行xvc0, 并注释其它tty配置

co:2345:respawn:/sbin/agetty xvc0 9600 vt100-nav
#1:2345:respawn:/sbin/mingetty tty1
#2:2345:respawn:/sbin/mingetty tty2
#3:2345:respawn:/sbin/mingetty tty3
#4:2345:respawn:/sbin/mingetty tty4
#5:2345:respawn:/sbin/mingetty tty5
#6:2345:respawn:/sbin/mingetty tty6

在/etc/securetty增加一行

xvc0

修改完/etc/inittab后运行

init q

使之生效

再次连接虚拟机终端, 出现登录界面

# xm console 14
INIT: Sending processes the TERM signal

CentOS release 5.3 (Final)
Kernel 2.6.18-128.7.1.el5xen on an x86_64

vm02.zone101.bj4.company.com login:
CentOS release 5.3 (Final)
Kernel 2.6.18-128.7.1.el5xen on an x86_64

vm02.zone101.bj4.company.com login:

至于为何没有自动添加参数, 原因不明, 有可能是BUG
Bug 463855 - No login prompt after headless serial console installation with virtualization
Bug 448429 - xm console requires DomU getty modification to point to xvc0

-fin-

multitail

2009-09-02T19:09:00.002+08:00

multitail

multitail是一个小工具, 于tail和watch命令相比, 它的好处是可以用来同时查看多个文件或命令的输出, 还可以将多个文件或命令输出到一个窗口显示, 定义配色方案彩色显示, 按模式匹配关键字过滤或高亮显示.
最新版本是2008-05-19发布的v5.2.2
从源码编译安装或从rpmforge上rpm包直接安装

使用方法:

multitail file1 file2 ...

比如:

multitail /var/log/messages /var/log/httpd/access_log

复杂一点的, 缓冲10000行, 窗口分割为2列, 反显ERR|dhcpd|twitter字符, 同时显示2个日志文件, 运行iostat命令, 每隔5秒运行ls命令

multitail -M 10000 -s 2 -EC "ERR|dhcpd|twitter" \
/var/log/messages \
/u01/blurlog/mysql/bqa2/cloud1/db01/logs/general.log \
-l "iostat 3" \
-R 5 -l "ls -l /var/log/messages"

在后台观察发现其实调用的就是一些tail命令

# ps -ef|grep 3454
root      3454 32310  0 10:45 pts/6    00:00:00 multitail -M 10000 -s 2 -EC ERR|dhcpd|twitter /var/log/messages /u01/blurlog/mysql/bqa2/cloud1/db01/logs/general.log -l iostat 3 -R 5 -l ls -l /var/log/messages
root      3455  3454  0 10:45 pts/6    00:00:00 tail --follow=name -n 62 /var/log/messages
root      3456  3454  0 10:45 pts/6    00:00:00 tail --follow=name -n 62 /u01/blurlog/mysql/bqa2/cloud1/db01/logs/general.log
root      3457  3454  0 10:45 pts/6    00:00:00 iostat 3
root      7440  3454  0 10:50 pts/6    00:00:00 multitail -M 10000 -s 2 -EC ERR|dhcpd|twitter /var/log/messages /u01/blurlog/mysql/bqa2/cloud1/db01/logs/general.log -l iostat 3 -R 5 -l ls -l /var/log/messages
root      7442  2641  0 10:50 pts/65   00:00:00 grep 3454

这个软件不是很稳定, 经常异常退出. 异常退出后, 必须手工杀掉后台的tail命令

外部链接:
MultiTail
用 multitail 查看许多文件 - 对话 UNIX: 适用于任何 UNIX 系统的 10 个出色的工具
Multitail
使用MultiTail同时监控多个文件

-fin-

how to find ip address of network interface

2009-08-27T19:27:00.002+08:00

how to find ip address assigned to network interface
如何取得网卡分配的IP地址

运行脚本:

/sbin/ifconfig -a|awk '
{
  if ( $0 ~ /^[^ ]/ ) {
    ifname=$1
    do {
      getline
      if ( $0 ~ /inet addr/ ) {
        ifaddr=substr($2, index($2, ":")+1)
      }
    } while ( $0 ~ /^[ ]/ )
    print ifname, ifaddr
    ifname=""
    ifaddr=""
  }
}
'

bond0 192.168.11.18
eth0
eth1
lo 127.0.0.1
sit0
tun0 172.16.220.94

列出系统中所有网卡对应的IPv4地址

-fin-

red herring

2009-08-27T12:17:00.001+08:00

red herring
熏鲱鱼

意指:
1. 故意引起误导, 转移注意的虚假线索
The way Barclay keeps his novels on the move is to put each book's protagonist up against a series of events that fall into a netherworld between clues and red herrings.
巴克莱推动小说情节发展的方式是设置真实和虚假的线索将故事中的主人公置于一系列陷入可怕的境地的事件中

2. 提出无关话题, 借以从原主题转移注意力的一种逻辑谬论
Cheney's 'Torture Works' Argument Is a Red Herring
切尼"虐囚有理"的争辩实属混淆视听

3. 成语: 难以名状的, 非驴非马, 不伦不类
neither fish nor fowl(flesh) (nor good red herring)
I guess that's why I'm a lukewarmer--neither fish nor fowl nor good red herring.
我想这就是为什么我是个温和主义者--非驴非马

起源:
通常认为, 起源于猎人在训练猎狗时用熏鲱鱼的气味干扰其追踪猎物.
事实并非如此, 这种说法最早出现在19世纪初一名英国记者写的一篇小说中, 用来隐喻抨击当时新闻界的错误报道, 由于这个比喻广泛流传深入人心, 后来被人误认为是一种训练猎狗的方法.

外部链接:
Red herring (idiom)
red herring
THE LURE OF THE RED HERRING

Fallacy: Red Herring
Red Herring

"NEITHER FISH NOR FLESH" AND ITS VARIATIONS*

Herring

-fin-

InnoDB Index Structure

2009-08-23T22:20:00.005+08:00

InnoDB Index Structure
InnoDB索引结构

题问:
查看下面查询语句, 字段b是数字类型, 建立了索引, 查询条件是LIKE, 为何使用了索引全扫描(type=index)?

mysql> desc t1;
+-------+---------+------+-----+---------+-------+
| Field | Type    | Null | Key | Default | Extra |
+-------+---------+------+-----+---------+-------+
| a     | int(11) | NO   | PRI | 0       |       |
| b     | int(11) | YES  | MUL | NULL    |       |
| c     | int(11) | YES  |     | NULL    |       |
+-------+---------+------+-----+---------+-------+
3 rows in set (0.01 sec)

mysql> explain select a, b from t1 where b like '50';
+----+-------------+-------+-------+---------------+------+---------+------+------+--------------------------+
| id | select_type | table | type  | possible_keys | key  | key_len | ref  | rows | Extra                    |
+----+-------------+-------+-------+---------------+------+---------+------+------+--------------------------+
|  1 | SIMPLE      | t1    | index | b             | b    | 5       | NULL | 4866 | Using where; Using index |
+----+-------------+-------+-------+---------------+------+---------+------+------+--------------------------+
1 row in set (0.00 sec)

mysql>

回答:
这条查询语句用到了覆盖索引(covering index), 跟InnoDB索引的结构有关.

1. 存储结构
InnoDB存储引擎中, 每一个表具有一个特殊的索引,叫聚集索引(clustered index, 又叫primary index), 行数据存储在该索引节点上, 这相当于Oracle的索引组织表的结构.
聚集索引建立的规则如下:
1)如果有主键, 那么选择该主键
2)如果没有主键, 有非空列的唯一索引, 则选择第一个非空唯一索引
3)否则, InnoDB存储引擎内部为每一条记录分配一个6字节的唯一标识(row id)
以此作为行的唯一标识创建聚集索引, 每条记录的数据存储在聚集索引的叶子节点上, 所以InnoDB聚集索引就是表本身.

其它索引称为二级索引(Secondary index)
二级索引的节点不仅保存索引键值(索引字段的值), 还保存了对应行的聚集索引(primary key)的键值. 此值起到指针的作用, 查询通过它去访问聚集索引, 扫描主键(primary key)从而得到行数据.(Oracle IOT的二级索引保存的是逻辑行标识符[logical rowid]). 这意味着, 通过二级索引查询要扫描两个索引, 二级索引和聚集索引

如图所示:

.
  primary key
         o  <-------
         |          |
      -------       |
     |       |      |
     o       o      |
     |       |      |
    ---     ---     |
   |   |   |   |    |
   o   o   o   o    |
  k+r k+r k+r k+r   |
                    |
  secondary key     |
         o          |
         |          |
      -------       |
     |       |      |
     o       o      |
     |       |      |
    ---     ---     |
   |   |   |   |    |
   o   o   o   o    |
  k+p k+p k+p k+p   |
                |   |
                 ---
.
primary key: 即primary index, 即聚集索引
secondary key: 即secondary index
k+r: key value + row data, 即索引键值和行数据
k+p: key value + primary key values, 即索引键值和主键键值

2. 选择主键
由于InnoDB索引存储结构的特殊性, 因此在设计表时对主键的选择显得格外重要, 主要原则有:
1) 显式指定主键, 不要让系统自动创建
如果不指定主键, 系统会自动创建一个自增长类型的隐含的字段(row id)作为主键, 长6字节
演示如下:
建一个表, 不带主键, 查看事务锁的状态

drop table if exists test;
create table test(a int, b int, c int) engine=innodb;
insert into test values(1, 16, 17), (2, 32, 34);
-- connection 1:
set transaction isolation level serializable;
start transaction;
select * from test;
-- connection 2:
set transaction isolation level serializable;
start transaction;
update test set a = 9;
-- connection 3:
SHOW ENGINE INNODB STATUS\G

------------
TRANSACTIONS
------------
Trx id counter 0 5673
Purge done for trx's n:o < 0 5672 undo n:o < 0 0
History list length 24
Total number of lock structs in row lock hash table 2
LIST OF TRANSACTIONS FOR EACH SESSION:
---TRANSACTION 0 0, not started, process no 4304, OS thread id 1162881344
MySQL thread id 20, query id 7423 localhost root
SHOW ENGINE INNODB STATUS
---TRANSACTION 0 5672, ACTIVE 3 sec, process no 4304, OS thread id 1162615104 starting index read
mysql tables in use 1, locked 1
LOCK WAIT 2 lock struct(s), heap size 368
MySQL thread id 19, query id 7422 localhost root Updating
update test set a = 9
------- TRX HAS BEEN WAITING 3 SEC FOR THIS LOCK TO BE GRANTED:
RECORD LOCKS space id 35 page no 3 n bits 72 index `GEN_CLUST_INDEX` of table `motoaccounts0/test` trx id 0 5672 lock_mode X waiting
Record lock, heap no 2 PHYSICAL RECORD: n_fields 6; compact format; info bits 0
 0: len 6; hex 000000002604; asc     & ;; 1: len 6; hex 000000001625; asc      %;; 2: len 7; hex 80000000340110; asc     4  ;; 3: len 4; hex 80000001; asc     ;; 4: len 4; hex 80000010; asc     ;; 5: len 4; hex 80000011; asc     ;;

------------------
---TRANSACTION 0 5670, ACTIVE 8 sec, process no 4304, OS thread id 1162348864
2 lock struct(s), heap size 368
MySQL thread id 18, query id 7419 localhost root

系统创建的索引的名字是GEN_CLUST_INDEX
索引结构中的字段个数(n_fields)为6, 第0个的长度是6字节(0: len 6;), 就是系统自动创建的隐含rowid; 后面两个1和2是系统内部使用的; 接下来第3,4,5代表表中的实际的字段

表如果带主键, 则是另一种情况

drop table if exists test;
create table test(a int primary key, b int, c int) engine=innodb;
insert into test values(1, 16, 17), (2, 32, 34);
-- connection 1:
set transaction isolation level serializable;
start transaction;
select * from test;
-- connection 2:
set transaction isolation level serializable;
start transaction;
update test set a = 9;
-- connection 3:
SHOW ENGINE INNODB STATUS\G

------------
TRANSACTIONS
------------
Trx id counter 0 5682
Purge done for trx's n:o < 0 5681 undo n:o < 0 0
History list length 27
Total number of lock structs in row lock hash table 2
LIST OF TRANSACTIONS FOR EACH SESSION:
---TRANSACTION 0 0, not started, process no 4304, OS thread id 1162881344
MySQL thread id 20, query id 7434 localhost root
SHOW ENGINE INNODB STATUS
---TRANSACTION 0 5681, ACTIVE 2 sec, process no 4304, OS thread id 1162615104 starting index read
mysql tables in use 1, locked 1
LOCK WAIT 2 lock struct(s), heap size 368
MySQL thread id 19, query id 7433 localhost root Searching rows for update
update test set a = 9
------- TRX HAS BEEN WAITING 2 SEC FOR THIS LOCK TO BE GRANTED:
RECORD LOCKS space id 36 page no 3 n bits 72 index `PRIMARY` of table `motoaccounts0/test` trx id 0 5681 lock_mode X waiting
Record lock, heap no 2 PHYSICAL RECORD: n_fields 5; compact format; info bits 0
 0: len 4; hex 80000001; asc     ;; 1: len 6; hex 00000000162e; asc      .;; 2: len 7; hex 800000002d0110; asc     -  ;; 3: len 4; hex 80000010; asc     ;; 4: len 4; hex 80000011; asc     ;;

------------------
---TRANSACTION 0 5679, ACTIVE 6 sec, process no 4304, OS thread id 1162348864
2 lock struct(s), heap size 368
MySQL thread id 18, query id 7430 localhost root

索引变成PRIMARY. n_fields值减少了一个, 变为5(n_fields 5;). 第0个是主键字段(0: len 4; hex 80000001; asc ;;).

2) 因为数据是按主键顺序存储的, 如果插入记录, 主键的值是随机的, 为了保证索引树的平衡, 聚集索引将重组, 这会产生较多写操作, 并导致产生索引碎片. 所以记录应该按主键升序顺序插入, 可以选择自增长类型字段作为主键.

3) 主键数据值应该尽可能小. 因为在所有索引中都包含了主键键值, 如果主键很大, 其它二级索引也会很大, 占用更多的空间. 如果自然主键很大, 可增加一个自增长列作主键.

4) 对主键字段更新操作也会引起聚集索引重组, 导致索引节点上的行数据的移动, 因此应尽量避免主键更新.

3. 问题解答

select a, b from t1 where b like '50';

语句查询了a,b两个字段, 字段b上有二级索引, 因为二级索引节点内包含了主键, 从二级索引中可以得到主键的值, 所以不用去查询表, 只要查询此二级索引一次就可以得到a,b字段的值. 执行计划的extra信息显示出"Using index", 表明可以从索引中取得所有要查询的列, 这种方式被称为索引覆盖(index covering).

外部链接:

13.6.10. InnoDB Table and Index Structures
InnoDB Internals: InnoDB File Formats and Source Code Structure
High performance MySQL By Baron Schwartz, Peter Zaitsev, Vadim Tkachenko, Jeremy D. Zawodny, Arjen Lentz, Derek Balling

How to exploit MySQL index optimizations
Everything you need to know about designing MySQL InnoDB primary keys
Index Layouts - Pro MySQL By Michael Kruckenberg, Jay Pipes
Innodb Performance Optimization

-fin-

1z0-043 lesson 12 Automatic Storage Management

2009-08-20T13:05:00.001+08:00

1z0-043 lesson 12

是否显示鄙人做的答案
是否显示标准答案

1.
QUESTION 14:
Consider the following configuration:
/devices/D1 is a member of disk group dgroupA.
/devices/D2 is a member of disk group dgroupA.
/devices/D3 is a member of disk group dgroupA.
You plan to add a new disk, /devices/D4, to then disk group dgroupA.
You execute the following command:

SQL> ALTER DISKGROUP dgroupA ADD DISK '/devices/D*';

Which task would be accomplished by the command?
A. The command adds the new disk, D4, to the disk group.
B. The command would result in an error because there is no disk by the name "'/devices/D*'"
C. The commnad will be ignored because disks starting with "D" are already members of the disk group.
D. The command would result in an error because no wildcard characters can be used in the disk name.
E. The command fisrt detaches all the member disks start with "D", and then reattaches them including the new disk.

A
见教材12-28

2.
QUESTION 16:
Immediately after adding a new disk to or removing an existing disk from an Automatic Storage Management(ASM) instance, you find that the performance of the database decreases initially, until the addition or removal process is completed. Performance then gradually returns to normal levels. Which two activities could you perform to maintain a consistent performance of the database while adding or removing disks?(Choose tow.)
A. increase the number of checkpoint processes
B. define the POWER option while adding or removing the disks
C. increase the number of DBWR processes by setting up a higher value for DB_WRITER_PROCESSES
D. increase the number of slave database writer processes by setting up higher value for DBWR_IO_SLAVES
E. increase the number of ASM Rebalance processes by setting up a higher value for ASM_POWER_LIMIT during the disk addition or removal

BE
E不对, 应该是减少才对

BE
见教材12-23
重新平衡会导致额外的IO, 影响了数据库性能, 所以应该是减小ASM_POWER_LIMIT

3.
QUESTION 19:
You are using an Automatic Storage Management(ASM) instance to manage the files of your production database. You have two disk groups, DG1 and DG2 with one device each. In the parameter file of the production database, the following parameters have been specified:
DB_CREATE_ONLINE_LOG_DEST_1='+dg1'
DB_CREATE_ONLINE_LOG_DEST_2='+dg2'
What would be the impact of this setting?
A. When a new log group is added, it would have one member in each disk group.
B. When a new log group is added, it would have two member in each disk group.
C. When a new tablespace is added, it would have one data file in each disk group.
D. When a new log file is added, it would have one member spread across the disk groups.

D
不知道

A
这个跟ASM没关系

4.
QUESTION 37:
Your production database uses an Automatic Storage Management(ASM) instance to manage its files. You want to add a new disk group to the ASM instance to manage the increased data load. What action would you perform to include the new disk group in the ASM instance without causing any impact on the currently connected users?
A. mount the new disk group in the ASM instance
B. restart the ASM instance and the production database instance
C. register the new disk groups in the production database instance
D. restart the ASM instance without restarting the production database instance
E. include the new disk group in the ASM_DISKSTRING parameter in the parameter file and restart the ASM instance

A
教材12-4
"This is done while the database is active."

5.
QUESTION 39:
In your production database you want to use an Automatic Storage Management(ASM) instance to manage the database files. Which option would you use to migrate the database files from a non-ASM instance to an ASM instance?
A. Oracle Migration Assistant
B. Recovery Manager(RMAN)
C. Oracle Data Pump Export and Import
D. conventional Oracle Export and Import
E. operating system uilities to copy the files to the ASM instance

B
教材12-34

6.
QUESTION 81:
You have a disk group, DGROUP1, with three disks and NORMAL redundancy. You execute the following command to create a template for the disk group:

ALTER DISKGROUP dgroup1
ADD TEMPLATE my_temp
ATTRIBUTES(MIRROR FINE);

Which statement is true?
A. When a file is created in DGROUP1 with the template, it would have three-way mirroring.
B. When a file is created in DGROUP1, the MY_TEMP template becomes the default template.
C. When a file is created in DGROUP1 with the template, it would have two-way mirroring and file striping.
D. When a file is created in DGROUP1 with the template, it would have three-way mirroring and file striping.
E. When a file is created in DGROUP1 with the template, it would have two-way mirroring but no file striping.

C
教材12-22, 缺省是两路镜像
PDF教材14-36

7.
QUESTION 91:
Consider the following command to add a new disk group called "tdgroupA" with two failover groups:

CREATE DISKGROUP tgroupA NORMAL REDUNDANCY
FAILOVERGROUP control01 DISK
'/devices/A1',
'/devices/A2',
'/devices/A3'
FAILOVERGROUP control02 DISK
'/devices/B1',
'/devices/B2',
'/devices/B3';

The disk "/devices/A1" is currently a member disk of a disk group by the name "tdgroup1".
Which task would be accomplished by the command?
A. This command would result in an error because a disk group can have only on failover group.
B. This command would result in an error because the /devices/A1 disk is a member of another disk group tdgroup1.
C. A new disk group called tdgroupA will be added with two failover groups and the /devices/A1 disk will get reattached to the new disk group without being detached from the existing one.
D. A new disk group called tdgroupA will be added with two failover groups and the /devices/A1 disk will be ignored for the new disk group becuase it is a member of an existing disk group tdgroup1.
E. A new disk group called tdgroupA will be added with two failover groups and the /devices/A1 disk gets detached from the existing disk group tdgroup1 and atteched to the new disk group tdgroupA.

B
不知道

B
教材12-27
FORCE indicates that a specified disk should be added to the specified disk group even though the disk is already formatted as a member of an ASM disk group.

8.
QUESTION 98:
Consider the following scenario:
You have a directory, data, under the disk group tdgroupA. You want to create an alias for on of the data files and you execute the following command:

ALTER DISKGROUP tdgroupA
ADD ALIAS '+tdgroupA/data/datafile.dbf'
FOR '+tdgroupA.231.45678';

Which task would be accomplished by the command?
A. The command drops the file +tdgroupA.231.45678.
B. The command physically relocated the file to +tdgroupA/data and renames the file to datafile.dbf.
C. The command creates a copy of the +tdgroupA.231.45678 file and places it in +tdgroupA/data after renaming the file to datafile.dbf.
D. The command creates an alias, datafile.dbf, and places it in +tdgroupA/data and does not remove the tdgroupA.231.45678 file.
E. The command creates a file, datafile.dbf, in +tdgroupA/data and removed the references for +tdgroupA.231.45678 from the data.

D
Oracle Database Administrator Guide Ch12 P25

PDF教材14-36

9.
QUESTION 108:
You have been assigned to manage a set of databases. The previous DBA did not leave you notes regarding the structure of each of the databases. While analyzing an instance, you notice that the system identifier(SID) for the instance is set to '+ASM'. What is the purpose for this instance?
A. This instance is being used to manage the operating system files.
B. This instance is being used to manage the files of other databases.
C. This instance is being used to manage the instances of other databases.
D. This instance is being used to manage the background processes of other instances.

B
教材12-9
DB_UNIQUE_NAME specifies the service provider name for which this ASM instance manages disk groups. The default value of +ASM must be modified only if you run multiple ASM instances on the same node.

10.
QUESTION 129:
Consider the following command to create a tablespace in your production database(which is using an Automatic Storage Management[ASM] instance to manage the database files):

CREATE TABLESPACE user_tbsp
DATAFILE '+dgroup3(user_temp)/user_files/user_tbsp' SIZE 200M;

What would be the result of this command?
A. It would result in an error because the template cannot be used along with the disk group.
B. It would result in an error because the path cannot be specified while creating a tablespace in a disk group.
C. It would create a tablespace with a data file that has an alias, and its attributes are set by the user-defined template.
D. It would create a tablespace with a data file that does not have an alias, and its attributes are set by the user-defined template.

C

PDF教材14-33讲了一点

11.
QUESTION 134:
Consider the following scenario:
You have a directory, data, under the disk group tdgroupA. You want to create an alias for one of the data files and you execute the following command:

ALTER DISKGROUP tdgroupA
ADD ALIAS '+tdgroupA/data/datafile.dbf'
FOR '+tdgroupA.231.45678';

Which task would be accomplished by the command?
A. The command drops the file +tdgroupA.231.45678
B. The command physically relocates the file to +tdgroupA/data and renames the file to datafile.dbf.
C. The command creates a copy of the +tdgroupA.231.45678 file and places it in +tdgroupA/data after renaming the file to datafile.dbf.
D. The command creates a synonym, datafile.dbf, and places it in +tdgroupA/data and dose not remove the +tdgroupA.231.45678 file.
E. The command creates a file, datafile.dbf, in +tdgroupA/data and removes the references for +tdgroupA.231.45678 from the data dictionary views.

D
Alias names(or just""aliases") are intended to provide a more use-friendly means of referring to ASM files, rather than using the system-generated filenames. You can create an alias for a file when you create it in the database, or you can add an alias to an existing file using the ADD ALIAS clause of the ALTER DISKGROUP statement. You can create an alias in any system-generated or user-created ASM directory. You cannot create an alias at the root level(+), however. The following statement adds a new alias name for a system-generated file name:

ALTER DISKGROUP dgroup1
ADD ALIAS '+dgroup1/mydir/second.dbf'
FOR '+dgroup1/sample/datafile/mytable.342.3';

REF.:Oracle(r) 10g Administrator Guide, 12-28

同第8题
PDF教材14-36

12.
QUESTION 152:
In your Automatic Storage Management(ASM) instance, one of the nonempty disk groups, DGROUP2, is no longer required and you want this disk group to be removed. You execute the following command to a achieve this objective:

DROP DISKGROUP dgroup1 EXCLUDING CONTENTS;

What would be the result of this command?
A. This command would result in an error because the disk group is not empty.
B. The command would distribute the contents of the specified disk group among all other disk groups and then drop the specified disk group.
C. The command would result in the contents being moved to the parent disk group and dropping of the disk group.
D. The command would result in the disk group being marked as INVALID because it cannot be dropped.
E. The command would drop the disk group, ignoring the EXCLUDING CONTENTS option.

A
有EXCLUDING这么用的吗?

A

有
Dropping Disk Groups
The default is EXCLUDING CONTENTS, which provides syntactic consistency and prevents you from dropping the disk group if it has any contents
DROP DISKGROUP
Specify EXCLUDING CONTENTS to ensure that Automatic Storage Management drops the disk group only when the disk group is empty. This is the default. If the disk group is not empty, an error will be returned.

13.
QUESTION 178:
You have an Automatic Storage Management(ASM) instance managing the database files of your production database. The database contains ASM files and non-ASM files. Using Recovery Manager(RMAN), you migrate the complete production database to use the ASM instance. Which statement is true?
A. RMAN would relocate all the data files to an ASM disk group and other files to an operating system location, defined using Oracle Managed Files(OMF).
B. RMAN would relocate the ASM files to an ASM disk group and the non-ASM files to an operating system location, defined using OMF.
C. RMAN would relocate all the database files to an ASM disk group.
D. RMAN would change the file definitions in the control file to use the ASM but would not relocate the database files physically.

C
教材12-34
Although files in a tablespace may be both ASM files and non-ASM files as a result of the tablespace history, RMAN commands enable non-ASM files to be relocated to an ASM disk group.

-----

14.
32. An ASM disk group can manage database files from how many different databases?
A. 1
B. 2
C. Limited only by disk space.
D. ASM disk groups manage tablespaces, not database files.

C
C. An ASM disk group can manage database files for essentially an unlimited number of different databases. Creating ASM disk groups is discussed in Chapter 9.

教材12-29
However, a disk group may contain files belonging to several databases, and a single database may use storage from multiple disk groups.

15.
33. High redundancy disk groups must have how many failure groups? (Choose the best answer.)
A. 1, because high redundancy disk groups rely on external RAID hardware or software
B. 2
C. Exactly 3
D. 3 or more

D
D. High redundancy disk groups require disks in at least three failure groups, but they can contain more if a higher level of redundancy or performance is desired. Controlling disk group redundancy is detailed in Chapter 9.

教材12-22
High redundancy:
Three-way mirroring
At least three failure groups

16.
34. Automatic Storage Management (ASM) disk group mirroring is done at which level?
A. Tablespace level
B. Extent level
C. Segment level
D. Datafile level

B
B. Disk group mirroring for ASM is done at the extent level. To learn about Automatic Storage Management mirroring, see Chapter 9.

说的不准确, 应该是以分配单元(AU)为单位

PDF教材14-21, (这个教材比较老, 所以说的不准确)
ASM does not mirror disks; rather, it mirrors extents.

教材12-20
ASM always spreads files evenly in 1 MB allocation unit (AU) chunks across all the disks in a disk group.
教材12-22
ASM does not mirror disks; rather, it mirrors allocation units.

17.
35. Identify two valid types of Oracle instances.
A. RMAN
B. DSKMGR
C. ASM
D. DBMS
E. RDBMS

CE
C, E. ASM instances manage ASM disk groups, and RDBMS instances are the traditional and only type of instance available before Oracle Database 10g. Configuring ASM and RDBMS instances for Automatic Storage Management is covered in Chapter 9.

教材12-9
INSTANCE_TYPE should be set to ASM for ASM instances.
教材12-10
INSTANCE_TYPE defaults to RDBMS and specifies that this instance is an RDBMS instance.

---

18.
1. Extents in an ASM file are allocated in units of which size?
A. 100KB
B. 10MB
C. 1MB
D. 64KB

C
忘了

C
C. ASM disks are partitioned in allocation units of one megabyte each.

教材12-20
ASM always spreads files evenly in 1 MB allocation unit (AU) chunks across all the disks in a disk group.

19.
2. To prevent connections to an ASM instance, you can use which of the following commands? (Choose the best answer.)
A. ALTER SYSTEM ENABLE RESTRICTED SESSION
B. SHUTDOWN IMMEDIATE
C. ALTER SYSTEM DISABLE CONNECTIONS
D. ALTER DATABASE ENABLE RESTRICTED SESSION

A
A. Similar to an RDBMS instance, you can use ALTER SYSTEM ENABLE RESTRICTED SESSION to prevent connections to the instance. While SHUTDOWN IMMEDIATE will prevent connections to the ASM instance, this is most likely overkill if all you want to do is temporarily prevent connections. Choices C and D are not valid commands and will generate an error message.

PDF教材14-8
Finally, you can prevent database instances from connecting to an ASM instance. When the command ALTER SYSTEM ENABLE RESTRICTED SESSION is issued to an ASM instance, database instances cannot connect to that ASM instance. Conversely, ALTER SYSTEM DISABLE RESTRICTED SESSION enables connections from database instances. This command permits an ASM instance to start up and mount disk groups for purposes of maintenance without allowing database instances to access the disk groups.

20.
3. Which initialization parameter in an ASM instance specifies the disk groups to be automatically mounted at instance startup?
A. ASM_DISKMOUNT
B. ASM_DISKGROUP
C. ASM_DISKSTRING
D. ASM_MOUNTGROUP

B
B. The initialization parameter ASM_DISKGROUP, valid only in an ASM instance, specifies the disk groups to be automatically mounted when the ASM instance starts. ASM_DISKSTRING is operating system dependent and restricts the file system devices that can be used to create disk groups. ASM_DISKMOUNT and ASM_MOUNTGROUP are not valid initialization parameters.

教材12-9
ASM_DISKGROUPS is the list of names of disk groups to be mounted by an ASM instance at startup, or when the ALTER DISKGROUP ALL MOUNT command is used.

21.
4. Which of the following command options is not valid for an ASM instance?
A. STARTUP OPEN
B. STARTUP NOMOUNT
C. STARTUP MOUNT
D. STARTUP OPEN RESTRICT
E. SHUTDOWN ABORT

A
D也不行?

A
A. An ASM instance can be started up and shut down in the same way that an RDBMS database can, except that an ASM instance cannot be in the OPEN state because it does not have a data dictionary or a control file.

为啥D可以?

22.
5. When an ASM instance receives a SHUTDOWN NORMAL command, what command does it pass on to all database instances that rely on the ASM instance’s disk groups?
A. TRANSACTIONAL
B. IMMEDIATE
C. ABORT
D. NORMAL
E. None of the above

D
D. When an ASM instance receives a SHUTDOWN command, it passes the same option (NORMAL, IMMEDIATE or TRANSACTIONAL) to all database instances that rely on the ASM instance for disk group services.

PDF教材14-18
Upon receiving the shutdown command, the ASM instance forwards the shutdown command with the same shutdown mode (NORMAL, IMMEDIATE, TRANSACTIONAL) to all database instances dependent on the ASM instance.

23.
6. When creating a disk group, what keyword must be specified if you need to reuse a disk that has previously been used as part of another disk group?
A. NOFORCE
B. REUSE
C. USE
D. FORCE
E. INCLUDING CONTENTS

D
D. You must use FORCE if the disk has previously been used as part of a disk group. If the disk has never been used as part of a disk group, using the FORCE keyword returns an error.

教材12-27
FORCE indicates that a specified disk should be added to the specified disk group even though the disk is already formatted as a member of an ASM disk group.

24.
7. Which of the following ASM file templates is not striped as fine?
A. FLASHBACK
B. ARCHIVELOG
C. CONTROLFILE
D. ONLINELOG

A
或者是B?

B
B. Files such as ARCHIVELOG files use coarse-grained striping. Fine striping stripes the files every 128KB while coarse striping stripes the files every 1MB. All file types with the exception of FLASHBACK, CONTROLFILE, and ONLINELOG are striped coarse.

见PDF教材14-35表格

25.
8. You want to migrate your database to ASM, so you’ve done a clean shutdown, made a closed backup of the entire database, noted the location of your control files and online redo log files, and changed your SPFILE to use OMF. The last step is running an RMAN script to do the conversion. Using the following steps, what is the correct order in which the following RMAN commands should be executed?

1. STARTUP NOMOUNT
2. ALTER DATABASE OPEN RESETLOGS
3. SQL "ALTER DATABASE RENAME 'logfile1 path' TO '+dgrp4 '"   # plus all other log files
4. SWITCH DATABASE TO COPY
5. BACKUP AS COPY DATABASE FORMAT '+dgrp4'
6. ALTER DATABASE MOUNT
7. RESTORE CONTROLFILE FROM 'controlfile_location'

A. 2, 5, 3, 1, 7, 6, 4
B. 1, 7, 6, 5, 4, 3, 2
C. 5, 1, 2, 7, 4, 6, 3
D. 7, 3, 1, 5, 6, 2, 4

B
B. After the RMAN script is run and the database is up and running successfully, you may delete the old database files.

教材12-34

26.
9. How can you reverse the effects of an ALTER DISKGROUP ... DROP DISK command if it has not yet completed?
A. Issue the ALTER DISKGROUP ... ADD DISK command.
B. Issue the ALTER DISKGROUP ... UNDROP DISKS command.
C. Issue the ALTER DISKGROUP ... DROP DISK CANCEL command.
D. Retrieve the disk from the recycle bin after the operation completes.

B
B. If the DROP DISK operation has not yet completed, you can cancel and roll back the entire DROP DISK operation by using ALTER DISKGROUP ... UNDROP DISKS, with the disk group still being continuously available to all users.

教材12-30
The third statement shows how to cancel a disk drop operation. The UNDROP command operates only on pending drops of disks; it has no effect on drops that have completed.

27.
10. To reference existing ASM files, you need to use a fully qualified ASM filename. Your development database has a disk group named DG2A, the database name is DEV19, and the ASM file that you want to reference is a datafile for the USERS02 tablespace. Which of the following is a valid ASM filename for this ASM file?
A. dev19/+DG2A/datafile/users02.701.2
B. +DG2A/dev19/datafile/users02.701.2
C. +DG2A/dev19/users02/datafile.701.2
D. +DG2A.701.2
E. +DG2A/datafile/dev19.users.02.701.2

B
好像D也是合法的?

B
B. A fully qualified existing ASM filename has the format +group/dbname/filetype/tag.file.incarnation. In this case, filetype is datafile, and tag is the tablespace name to which it belongs, or users02.

题目中问的是全称(fully qualified name), 所以不选D
PDF教材14-32

28.
11. Which background process coordinates the rebalance activity for disk groups?
A. ORBn
B. OSMB
C. RBAL
D. ASMn

C
C. RBAL coordinates rebalance activity for a disk group in an ASM instance, ORBn actually performs the extent movement in an ASM instance, and OSMB acts as a bridge between the ASM instance and the RDBMS instance. There is no such process name ASMn.

教材12-5
One coordinates rebalance activity for disk groups. It is called RBAL.

29.
12. On the development database rac0 there are six raw devices: /dev/raw/raw1 through /dev/raw/raw6. /dev/raw/raw1 and /dev/raw/raw2 are 8GB each, and the rest are 6GB each. An existing disk group +DATA1, of NORMAL REDUNDANCY, uses /dev/raw/raw1 and /dev/raw/raw2. Which series of the following commands will drop one of the failure groups for +DATA1, create a new disk group +DATA2 using two of the remaining four raw devices, and then cancel the drop operation from +DATA1?
A.

ALTER DISKGROUP DATA1 DROP DISK DATA1_0001;
CREATE DISKGROUP DATA2 NORMAL REDUNDANCY
  FAILGROUP DATA1A DISK '/dev/raw/raw3'
  FAILGROUP DATA1B DISK '/dev/raw/raw4';
ALTER DISKGROUP DATA1 UNDROP DISKS;

ALTER DISKGROUP DATA1 DROP DISK DATA1_0001;
CREATE DISKGROUP DATA2 HIGH REDUNDANCY
  FAILGROUP DATA1A DISK '/dev/raw/raw3'
  FAILGROUP DATA1B DISK '/dev/raw/raw4'
ALTER DISKGROUP DATA1 UNDROP DISKS;

ALTER DISKGROUP DATA1 DROP DISK DATA1_0001;
CREATE DISKGROUP DATA2 NORMAL REDUNDANCY
  FAILGROUP DATA1A DISK '/dev/raw/raw3'
  FAILGROUP DATA1B DISK '/dev/raw/raw4';
ALTER DISKGROUP DATA1 UNDROP DATA1_0001;

ALTER DISKGROUP DATA1 DROP DISK DATA1_0001
  ADD DISKGROUP DATA2 NORMAL REDUNDANCY
  FAILGROUP DATA1A DISK '/dev/raw/raw3'
  FAILGROUP DATA1B DISK '/dev/raw/raw4';
ALTER DISKGROUP DATA1 UNDROP DISKS;

A
题意不清,B也对?

A
A. Note that the UNDROP operation will cancel a drop operation in progress but cannot reverse a drop operation that has already completed. For HIGH REDUNDANCY, at least three failure groups must be specified. While you can combine a drop and add operation into one command, the command can reference only one disk group.

高冗余度至少需要3个硬盘, 所以不选B

30.
13. Which type of database file is spread across all disks in a disk group?
A. All types of files are spread across all disks in the disk group.
B. Datafiles
C. Redo log files
D. Archived redo log files
E. Control files

A
A. All types of database files are spread across all disks in the disk group to ensure redundancy unless the redundancy is set to EXTERNAL.

教材12-19
ASM files are always spread across all the ASM disks in the disk group.

31.
14. How can you reverse the effects of an ALTER DISKGROUP ... DROP DISK command if it has already completed?
A. Issue the ALTER DISKGROUP ... ADD DISK command.
B. Issue the ALTER DISKGROUP ... UNDROP DISKS command.
C. Issue the ALTER DISKGROUP ... DROP DISK CANCEL command.
D. Retrieve the disk from the recycle bin after the operation completes.

A
A. If the DROP DISK operation has already completed, you must use ALTER DISKGROUP ... ADD DISK to add the disk back to the disk group. In any case, the disk group is continuously available to all users.

教材12-30

32.
15. Which of the following ALTER DISKGROUP commands does not use V$ASM_OPERATION to record the status of the operation?
A. ADD DIRECTORY
B. DROP DISK
C. RESIZE DISK
D. REBALANCE
E. ADD FAILGROUP

A
A. The ADD DIRECTORY command does not use V$ASM_OPERATION to track its progress, because this operation adds only a small amount of metadata—a directory object—to the disk group and takes a minimal amount of time to complete.

教材14-29
The ALTER DISKGROUP DROP, RESIZE, and REBALANCE commands return before the operation is complete. To monitor progress of these long-running operations, you can query the V$ASM_OPERATION fixed view.

33.
16. If you use ALTER DISKGROUP ... ADD DISK and specify a wildcard for the discovery string, what happens to disks that are already a part of the same or another disk group?
A. The command fails unless you specify the FORCE option.
B. The command fails unless you specify the REUSE option.
C. The command must be reissued with a more specific discovery string.
D. The other disks already part of the disk group are ignored.

D
D. The ALTER DISKGROUP ... ADD DISK command adds all disks that match the discovery string but are not already part of the same or another disk group.

教材12-28
The second command adds A4 to the DGROUPA disk group. It ignores the other disks, even though they match the discovery string, because they are already part of the DGROUPA disk group.

34.
17. Choose the set of the following initialization parameters that is valid and recommended for an ASM instance.
A. INSTANCE_TYPE=RDBMS
ASM_POWER_LIMIT=2
LARGE_POOL_SIZE=8MB
DB_UNIQUE_NAME=+ASM
ASM_DISKGROUPS=DATA1,DATA2
B. INSTANCE_TYPE=ASM
ASM_POWER_LIMIT=2
LARGE_POOL_SIZE=8MB
DB_UNIQUE_NAME=+ASM
ASM_DISKGROUPS=DATA1,DATA2
C. INSTANCE_TYPE=ASM
ASM_POWER_LIMIT=15
LARGE_POOL_SIZE=8MB
DB_UNIQUE_NAME=+ASM
ASM_DISKGROUPS=DATA1,DATA2
D. INSTANCE_TYPE=ASM
ASM_POWER_LIMIT=2
LARGE_POOL_SIZE=4MB
DB_UNIQUE_NAME=+ASM
ASM_DISKGROUPS=DATA1,DATA2

B
B. The INSTANCE_TYPE for an ASM instance is ASM; otherwise, it is RDBMS, whether it uses ASM or not. The ASM_POWER_LIMIT command controls the speed of a disk group rebalance, but its maximum value is 11. For an ASM instance, the minimum recommended value for LARGE_POOL_SIZE is 8MB.

教材12-9

35.
18. Which of the following scenarios concerning ASM instance shutdown is correct?
A. When an ASM instance is shut down with NORMAL, IMMEDIATE, or TRANSACTIONAL, the same shutdown command is passed to the dependent instances and the ASM instance waits for all dependent instances to shut down before it shuts down.
B. When an ASM instance shuts down with NORMAL, an alert is sent to all dependent instances, notifying the DBA to shut down the dependent instances manually before the ASM instance shuts down.
C. When an ASM instance shuts down with the TRANSACTIONAL option, all dependent instances shut down with NORMAL, IMMEDIATE, or TRANSACTIONAL, depending on the dependent database’s default.
D. When an ASM instance is shut down with NORMAL, IMMEDIATE, or TRANSACTIONAL, the same shutdown command is passed to the dependent instances and the ASM instance does not wait for all dependent instances to shut down before it shuts down.
E. When an ASM instance shuts down with the IMMEDIATE option, the ASM instance shuts down immediately and all dependent instances shut down with ABORT.

A
A. When an ASM instance shuts down with NORMAL, IMMEDIATE, or TRANSACTIONAL, the same shutdown option is passed to all dependent instances and the ASM instance waits for the dependent instances to shut down before shutting itself down. If an ASM instance shuts down with ABORT, it immediately shuts down, the dependent instances lose their connection to the ASM instance, and as a result, they shut down with ABORT either before or after the ASM instance shuts down completely.

PDF教材14-18

36.
19. A database can create datafiles in how many different disk groups? (Choose the best answer.)
A. Each datafile in the database can reside in a different disk group.
B. One
C. Disk groups manage tablespaces, not datafiles.
D. A maximum of two, one for SYSTEM and SYSAUX and the other tablespaces in another disk group.

A
A. Each database datafile can reside in a different disk group; each disk group can also contain datafiles from other databases.

教材12-19

37.
20. ASM supports all of the following file types except for which of the following? (Choose all that apply.)
A. Database files
B. SPFILEs
C. Redo log files
D. Archived log files
E. RMAN backup sets
F. Password files
G. init.ora files

FGB

FG
F, G. ASM supports datafiles, log files, control files, archive logs, RMAN backup sets, SPFILEs, and other Oracle database file types, but not password files or init.ora files.

PDF教材14-34表格

-fin-

gone fishing

2009-08-13T23:55:00.002+08:00

gone fishing

去钓鱼就是指去度假(vacation)了, 大概因为美国人度假时比较喜欢钓鱼?
有旷工的(absent from duty),关闭的(closed),停止服务(out of service)的意思

例句:
One Republican congressman complained that "it was hard for Mr. Bush to get his message out if the White House lectern had a ‘Gone Fishing' sign on it."

Gone Fishin'. Okay, not really. But after some consideration, I’ve decided to take the summer off from blogging.

I've been looking through the documentation, and I don't seem to see mentioned anywhere how I can setup auto-replies, or canned replies on an account by account basis. I seem to remember having done this on other mailer systems using a .gone-fishing or .vacation file.

-fin-

Slow queries issue

2009-08-13T14:29:00.004+08:00

Troubleshooting case study: Slow queries issue

提问1
Focus on the queries of this form:

### 423 Queries 
### Total time: 13459, Average time: 31.8179669030733
### Taking 13  to 74  seconds to complete
### Rows analyzed 0 - 38
use db1;
SELECT *     FROM t_u_sch     WHERE s_time <= XXX     ORDER BY s_time ASC     LIMIT XXX     FOR UPDATE;

use db1;
SELECT *     FROM t_u_sch     WHERE s_time <= 1234567890     ORDER BY s_time ASC     LIMIT 38     FOR UPDATE;

These are scheduler related queries in the DB1 for FB, Twitter, MySpace, etc. We’re selecting a bunch of accounts to schedule work items for. So we select then update the last scheduled sync time column. We also insert new accts in these tables.

回答1
it's still hard to tell the root cause at the moment. but we found index of t_u_sch is fragmented,

mysql> show table status like
    -> 't_u_sch';
+-------------------+--------+---------+------------+------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+-------------+------------+-----------------+----------+----------------+----------------------------------------------------------------------------------+
| Name              | Engine | Version | Row_format | Rows | Avg_row_length | Data_length | Max_data_length | Index_length | Data_free | Auto_increment | Create_time         | Update_time | Check_time | Collation       | Checksum | Create_options | Comment                                                                          |
+-------------------+--------+---------+------------+------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+-------------+------------+-----------------+----------+----------------+----------------------------------------------------------------------------------+
| t_u_sch           | InnoDB |      10 | Compact    | 1140 |            172 |      196608 |               0 |    336592896 |         0 |           NULL | 2009-06-29 23:59:29 | NULL        | NULL       | utf8_general_ci |     NULL |                | InnoDB free: 3072 kB; (`b_a_id      `) REFER `db1/t_u     `(`b_a_id      `) ON D |
+-------------------+--------+---------+------------+------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+-------------+------------+-----------------+----------+----------------+----------------------------------------------------------------------------------+
1 row in set (0.13 sec)

as you can see index length is abnormally larger than the data length, it may be caused by frequently deletion and insertion on table. when an index becomes fragmented, index scan query will take longer time. to defrag it and free up spaces, you can use "optimize table ..." or "alter table ... engine=innodb"

mysql> alter table t_u_sch ENGINE=INNODB;
Query OK, 763 rows affected (0.25 sec)
Records: 763  Duplicates: 0  Warnings: 0

mysql> show table status like 't_u_sch';
+-------------------+--------+---------+------------+------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+-------------+------------+-----------------+----------+----------------+----------------------------------------------------------------------------------+
| Name              | Engine | Version | Row_format | Rows | Avg_row_length | Data_length | Max_data_length | Index_length | Data_free | Auto_increment | Create_time         | Update_time | Check_time | Collation       | Checksum | Create_options | Comment                                                                          |
+-------------------+--------+---------+------------+------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+-------------+------------+-----------------+----------+----------------+----------------------------------------------------------------------------------+
| t_u_sch           | InnoDB |      10 | Compact    |  871 |            131 |      114688 |               0 |        49152 |         0 |           NULL | 2009-08-06 10:46:00 | NULL        | NULL       | utf8_general_ci |     NULL |                | InnoDB free: 0 kB; (`b_a_id      `) REFER `db1/t_u     `(`b_a_id      `) ON DELE |
+-------------------+--------+---------+------------+------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+-------------+------------+-----------------+----------+----------------+----------------------------------------------------------------------------------+
1 row in set (0.00 sec)

other *_u_sch* tables also need to be defragmented by this way.

thanks.

发现有些InnoDB表的索引很大, 而且随着时间推移不断的增大, 与实际数据量很不相符, 实际只有1000条左右的记录.
对这个表的操作主要是大量的UPDATE, 怀疑索引因此产生了碎片或平衡树的结构"失衡"了(Oracle数据库中也有这种情况)
但不能确定, 也不知道如何在测试系统上重现此错误(测试时发现如果不提交, UPDATE操作确实会导致索引一直变大, 但提交后, 发现过一段时间MySQL会自动收缩索引, 可以看到index_length变小)
临时的解决方法是, 定期优化表(optimize table ...), 达到重建索引, 释放空间的目的

提问2

We are continuing to get slow queries and used "SHOW TABLE STATUS' to see if there is index fragmentation. For some tables, I am finding that 'Index_length' is zero even though there are indexes on the table. This is happening on db.some.where.com.

For example:

# Time: 090806 21:48:12
# User@Host: user1[user1] @ a1.some.where.com [12.34.56.78]
# Query_time: 7  Lock_time: 0  Rows_sent: 6  Rows_examined: 6
use db1;
SELECT * FROM t_s_ch      WHERE b_a_id = '123456ABC' AND           s_a >= 28924     ORDER BY s_a ASC     LIMIT 2147483647;

mysql> show table status like 't_s_ch';
+-------------------+--------+---------+------------+---------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+-------------+------------+-----------------+----------+----------------+----------------------------------------------------------------------------------+
| Name              | Engine | Version | Row_format | Rows    | Avg_row_length | Data_length | Max_data_length | Index_length | Data_free | Auto_increment | Create_time         | Update_time | Check_time | Collation       | Checksum | Create_options | Comment                                                                          |
+-------------------+--------+---------+------------+---------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+-------------+------------+-----------------+----------+----------------+----------------------------------------------------------------------------------+
| t_s_ch            | InnoDB |      10 | Compact    | 9462693 |            164 |  1559756800 |               0 |            0 |         0 |           NULL | 2009-04-24 17:21:19 | NULL        | NULL       | utf8_general_ci |     NULL |                | InnoDB free: 45056 kB; (`b_a_id      `) REFER `db1/f_u     `(`b_a_id      `) ON  | 
+-------------------+--------+---------+------------+---------+----------------+-------------+-----------------+--------------+-----------+----------------+---------------------+-------------+------------+-----------------+----------+----------------+----------------------------------------------------------------------------------+
1 row in set (0.70 sec)

Is there a reason why 'Index_length' would be zero? I am checking Google and I haven't found a reason so far. Does this table not have an index anymore?

回答2

I think it's an expected behavior. In InnoDB, every table must have a clustered index, data records are stored as leaf nodes of this index, which means clustered index is the data records, it is counted as data_length not index_length. In your case, the primacy key is the clustered index, and no other indexes exist, so index_length is zero.

InnDB表都必须有一个聚集索引(clustered index), 记录数据存储在这个聚集索引中, 类似于Oracle的索引组织表(index-organized table).
如果表上定义了主键, 则使用主键作为聚集索引
否则选择第一个非空的唯一索引作为聚集索引
如果都没有, 则内部创建一个隐藏的rowid单调递增列, 基于该列创建隐藏的聚集索引
(参考13.6.10.1. Clustered and Secondary Indexes)

所以聚集索引也就是记录数据, 算作data_length,而不是index_length

回答3

1.
from mysqladmin status output and slow query log, it seems there are some slow queries using full table scans

so we suggest:
1)add an index on customer_tab(custid,deptid)

2)extend the index in con_gr_tab/con_tab/con_id_tab/con_tab2 from (cust_id) to (cust_id,sn)

3)for following select statement, it is better to use "=" for comparision rather than "like"

SELECT d.deptid, d.customid, d.sn, d.version, d.email, d.dmi, d.area, d.provider, d.manufacturer, d.nation, d.number FROM custominfo d WHERE d.deptid LIKE '1234567890' LIMIT 100;

4)change dept_id in db_a_tab/db_base_tab/db_queue_tab/db_b_tab from varchar(80) to bigint(20)

5)decrease the size of cust_id in customs_tab from varchar(512) to varchar(80)

2.
in dbs, general/slow query logs are growing huge,

[db1.some.where.com]
-rw-rw---- 1 mysql mysql           0 Aug 11 15:56 /mysql_logs/error.log
-rw-rw---- 1 mysql mysql 41254683176 Aug 12 06:41 /mysql_logs/general.log
-rw-rw---- 1 mysql mysql  2117239370 Aug 12 06:41 /mysql_logs/slow-queries.log
[db2.some.where.com]
-rw-rw---- 1 mysql mysql           0 Aug 11 15:56 /mysql_logs/error.log
-rw-rw---- 1 mysql mysql 32935022398 Aug 12 06:41 /mysql_logs/general.log
-rw-rw---- 1 mysql mysql   458081312 Aug 12 06:41 /mysql_logs/slow-queries.log
[db3.some.where.com]
-rw-rw---- 1 mysql mysql           0 Aug 11 15:56 /mysql_logs/error.log
-rw-rw---- 1 mysql mysql 30089483757 Aug 12 06:41 /mysql_logs/general.log
-rw-rw---- 1 mysql mysql  1156294323 Aug 12 06:41 /mysql_logs/slow-queries.log
[db4.some.where.com]
-rw-rw---- 1 mysql mysql           0 Aug 11 15:56 /mysql_logs/error.log
-rw-rw---- 1 mysql mysql 29886541601 Aug 12 06:41 /mysql_logs/general.log
-rw-rw---- 1 mysql mysql  1871892345 Aug 12 06:41 /mysql_logs/slow-queries.log

these logs need to be cleaned up, logrotate can do this, for example:

# cat /etc/logrotate.d/mysql
/mysql/logs/error.log /mysql/logs/general.log /mysql/logs/slow-queries.log {
  daily
  create 0660 mysql mysql
  notifempty
  #size 100M
  missingok
  nocompress
  rotate 120
  sharedscripts
  postrotate
    /usr/bin/mysqladmin flush-logs
  endscript
}
 
# logrotate -f /etc/logrotate.d/mysql

it is also recommended to turn on general query log only when needed, since it is probably an impact on performance. to disable it, a restart of mysql server is required.
if using mysql 5.1, it can be disabled/enabled dynamically without restarting mysql.

3. for better innodb performance, increase innodb_buffer_pool_size, for example to 6g. in general, it can be set up to 80% of phsical memory size.

1.使用命令"mysqladmin extented-status -r -i60", 查看指定时间间隔内的数据库状态值的变化情况
Select_scan,Sort_scan说明存在全表扫描
通过查看慢速查询日志(slow query log), 能够发现运行时间超过指定时长的SQL语句
2.Linux自带Logrotate工具实现Mysql查询日志的归档
3.InnoDB缓冲池最好设大些, 比如通常可以设置为物理内存的80%

-fin-

Rotating MySQL log files

2009-08-11T17:03:00.006+08:00

Rotating MySQL log files
轮换MySQL日志文件

用Logrotate维护MySQL日志

通常打开MySQL的普通查询日志(general query log)和慢速查询日志(slow query log), 以便查找(可能)出现的问题
在繁忙的系统中, 日志文件增长很快, 对大文件查询起来不方便, 写日志文件本身也会影响磁盘性能

MySQL 5.0中普通/慢速查询日志参数是个静态的参数,只能通过重启MySQL服务生效
MySQL 5.1版本后普通/慢速查询日志参数是动态的, 能够动态的启用或关闭. 可以平时关闭普通查询日志, 当需要时再打开

当日志文件增长过大时需要截断(truncate), 使用命令"> logfile"
此命令清空文件全部内容,而不删除文件

# ls -l install.log.bak
-rw-r--r-- 1 root root 17244 Jun  1 09:29 install.log.bak
# >install.log.bak
# ls -l install.log.bak
-rw-r--r-- 1 root root 0 Aug 11 06:45 install.log.bak
#

截断日志文件后以前的内容就没有了, 所以这并非是一个理想的方案, 需要的是对日志进行归档, 保留旧的日志, 产生新的日志文件
方法见MySQL参考手册:

shell> mv host_name.log host_name-old.log
shell> mysqladmin flush-logs
shell> cp host_name-old.log backup-directory
shell> rm host_name-old.log

日志文件改名, 这不会使MySQL服务终止, 所以不影响在线业务
运行flush-logs命令, 刷新日志文件(包括关闭/重新打开查询日志文件)
备份旧的日志文件
删除旧的日志文件

使用linux系统中logrotate自动工作, 配置文件如下:

cat >/etc/logrotate.d/mysql <<'EOF'
/mysql/logs/error.log /mysql/logs/general.log /mysql/logs/slow-queries.log {
  daily
  create 0660 mysql mysql
  notifempty
  #size 100M
  missingok
  nocompress
  rotate 120
  sharedscripts
  postrotate
    if test -x /usr/bin/mysqladmin && \
       /usr/bin/mysqladmin ping &>/dev/null
    then
       /usr/bin/mysqladmin flush-logs
    fi
  endscript
}
EOF

daily: 每天归档一次
create 0660 mysql mysql: 归档后创建新文件
notifempty: 文件为空时不做处理
#size 100M: 文件大小超过100M也归档
missingok: 文件不存在时不报错
nocompress: 不压缩归档文件
rotate 120: 保留120个归档文件
sharedscripts: 共享的脚本,意味着对所有文件只运行一次
postrotate: 轮换后要运行的脚本

试运行一下

/usr/sbin/logrotate -vf /etc/logrotate.d/mysql

查看结果

[root@db01 /mysql/logs]# ll
total 1675432
-rw-rw---- 1 mysql mysql        472 Aug 11  2009 error.log
-rw-rw---- 1 mysql mysql        472 Aug 11 15:03 error.log.1
-rw-rw---- 1 mysql mysql        472 Aug 11 14:57 error.log.2
-rw-rw---- 1 mysql mysql          0 Aug 11 15:03 error.log-old
-rw-rw---- 1 mysql mysql        687 Aug 11  2009 general.log
-rw-rw---- 1 mysql mysql       1082 Aug 11 15:03 general.log.1
-rw-rw---- 1 mysql mysql      37115 Aug 11 15:01 general.log.2
-rw-rw---- 1 mysql mysql 1693930999 Aug 11 11:06 general.log.3
-rw-rw---- 1 mysql mysql        354 Aug 11  2009 slow-queries.log
-rw-rw---- 1 mysql mysql        354 Aug 11 15:03 slow-queries.log.1
-rw-rw---- 1 mysql mysql   18263218 Aug 11 14:57 slow-queries.log.2
[root@db01 /mysql/logs]# /usr/sbin/logrotate -v /etc/logrotate.d/mysql
reading config file /etc/logrotate.d/mysql
reading config info for /mysql/logs/error.log /mysql/logs/general.log /mysql/logs/slow-queries.log

Handling 1 logs

rotating pattern: /mysql/logs/error.log /mysql/logs/general.log /mysql/logs/slow-queries.log  after 1 days (120 rotations)
empty log files are not rotated, old logs are removed
considering log /mysql/logs/error.log
  log does not need rotating
considering log /mysql/logs/general.log
  log does not need rotating
considering log /mysql/logs/slow-queries.log
  log does not need rotating
not running postrotate script, since no logs were rotated
[root@db01.cloud2.bqa3.blurdev.com /mysql/logs]# /usr/sbin/logrotate -vf /etc/logrotate.d/mysql
reading config file /etc/logrotate.d/mysql
reading config info for /mysql/logs/error.log /mysql/logs/general.log /mysql/logs/slow-queries.log

Handling 1 logs

rotating pattern: /mysql/logs/error.log /mysql/logs/general.log /mysql/logs/slow-queries.log  forced from command line (120 rotations)
empty log files are not rotated, old logs are removed
considering log /mysql/logs/error.log
  log needs rotating
considering log /mysql/logs/general.log
  log needs rotating
considering log /mysql/logs/slow-queries.log
  log needs rotating
rotating log /mysql/logs/error.log, log->rotateCount is 120
renaming /mysql/logs/error.log.120 to /mysql/logs/error.log.121 (rotatecount 120, logstart 1, i 120),
old log /mysql/logs/error.log.120 does not exist
renaming /mysql/logs/error.log.119 to /mysql/logs/error.log.120 (rotatecount 120, logstart 1, i 119),
old log /mysql/logs/error.log.119 does not exist
renaming /mysql/logs/error.log.118 to /mysql/logs/error.log.119 (rotatecount 120, logstart 1, i 118),
old log /mysql/logs/error.log.118 does not exist
renaming /mysql/logs/error.log.117 to /mysql/logs/error.log.118 (rotatecount 120, logstart 1, i 117),
old log /mysql/logs/error.log.117 does not exist
renaming /mysql/logs/error.log.116 to /mysql/logs/error.log.117 (rotatecount 120, logstart 1, i 116),
old log /mysql/logs/error.log.116 does not exist
renaming /mysql/logs/error.log.115 to /mysql/logs/error.log.116 (rotatecount 120, logstart 1, i 115),
old log /mysql/logs/error.log.115 does not exist
renaming /mysql/logs/error.log.114 to /mysql/logs/error.log.115 (rotatecount 120, logstart 1, i 114),
old log /mysql/logs/error.log.114 does not exist
renaming /mysql/logs/error.log.113 to /mysql/logs/error.log.114 (rotatecount 120, logstart 1, i 113),
old log /mysql/logs/error.log.113 does not exist
renaming /mysql/logs/error.log.112 to /mysql/logs/error.log.113 (rotatecount 120, logstart 1, i 112),
old log /mysql/logs/error.log.112 does not exist
renaming /mysql/logs/error.log.111 to /mysql/logs/error.log.112 (rotatecount 120, logstart 1, i 111),
old log /mysql/logs/error.log.111 does not exist
renaming /mysql/logs/error.log.110 to /mysql/logs/error.log.111 (rotatecount 120, logstart 1, i 110),
old log /mysql/logs/error.log.110 does not exist
renaming /mysql/logs/error.log.109 to /mysql/logs/error.log.110 (rotatecount 120, logstart 1, i 109),
old log /mysql/logs/error.log.109 does not exist
renaming /mysql/logs/error.log.108 to /mysql/logs/error.log.109 (rotatecount 120, logstart 1, i 108),
old log /mysql/logs/error.log.108 does not exist
renaming /mysql/logs/error.log.107 to /mysql/logs/error.log.108 (rotatecount 120, logstart 1, i 107),
old log /mysql/logs/error.log.107 does not exist
renaming /mysql/logs/error.log.106 to /mysql/logs/error.log.107 (rotatecount 120, logstart 1, i 106),
old log /mysql/logs/error.log.106 does not exist
renaming /mysql/logs/error.log.105 to /mysql/logs/error.log.106 (rotatecount 120, logstart 1, i 105),
old log /mysql/logs/error.log.105 does not exist
renaming /mysql/logs/error.log.104 to /mysql/logs/error.log.105 (rotatecount 120, logstart 1, i 104),
old log /mysql/logs/error.log.104 does not exist
renaming /mysql/logs/error.log.103 to /mysql/logs/error.log.104 (rotatecount 120, logstart 1, i 103),
old log /mysql/logs/error.log.103 does not exist
renaming /mysql/logs/error.log.102 to /mysql/logs/error.log.103 (rotatecount 120, logstart 1, i 102),
old log /mysql/logs/error.log.102 does not exist
renaming /mysql/logs/error.log.101 to /mysql/logs/error.log.102 (rotatecount 120, logstart 1, i 101),
old log /mysql/logs/error.log.101 does not exist
renaming /mysql/logs/error.log.100 to /mysql/logs/error.log.101 (rotatecount 120, logstart 1, i 100),
old log /mysql/logs/error.log.100 does not exist
renaming /mysql/logs/error.log.99 to /mysql/logs/error.log.100 (rotatecount 120, logstart 1, i 99),
old log /mysql/logs/error.log.99 does not exist
renaming /mysql/logs/error.log.98 to /mysql/logs/error.log.99 (rotatecount 120, logstart 1, i 98),
old log /mysql/logs/error.log.98 does not exist
renaming /mysql/logs/error.log.97 to /mysql/logs/error.log.98 (rotatecount 120, logstart 1, i 97),
old log /mysql/logs/error.log.97 does not exist
renaming /mysql/logs/error.log.96 to /mysql/logs/error.log.97 (rotatecount 120, logstart 1, i 96),
old log /mysql/logs/error.log.96 does not exist
renaming /mysql/logs/error.log.95 to /mysql/logs/error.log.96 (rotatecount 120, logstart 1, i 95),
old log /mysql/logs/error.log.95 does not exist
renaming /mysql/logs/error.log.94 to /mysql/logs/error.log.95 (rotatecount 120, logstart 1, i 94),
old log /mysql/logs/error.log.94 does not exist
renaming /mysql/logs/error.log.93 to /mysql/logs/error.log.94 (rotatecount 120, logstart 1, i 93),
old log /mysql/logs/error.log.93 does not exist
renaming /mysql/logs/error.log.92 to /mysql/logs/error.log.93 (rotatecount 120, logstart 1, i 92),
old log /mysql/logs/error.log.92 does not exist
renaming /mysql/logs/error.log.91 to /mysql/logs/error.log.92 (rotatecount 120, logstart 1, i 91),
old log /mysql/logs/error.log.91 does not exist
renaming /mysql/logs/error.log.90 to /mysql/logs/error.log.91 (rotatecount 120, logstart 1, i 90),
old log /mysql/logs/error.log.90 does not exist
renaming /mysql/logs/error.log.89 to /mysql/logs/error.log.90 (rotatecount 120, logstart 1, i 89),
old log /mysql/logs/error.log.89 does not exist
renaming /mysql/logs/error.log.88 to /mysql/logs/error.log.89 (rotatecount 120, logstart 1, i 88),
old log /mysql/logs/error.log.88 does not exist
renaming /mysql/logs/error.log.87 to /mysql/logs/error.log.88 (rotatecount 120, logstart 1, i 87),
old log /mysql/logs/error.log.87 does not exist
renaming /mysql/logs/error.log.86 to /mysql/logs/error.log.87 (rotatecount 120, logstart 1, i 86),
old log /mysql/logs/error.log.86 does not exist
renaming /mysql/logs/error.log.85 to /mysql/logs/error.log.86 (rotatecount 120, logstart 1, i 85),
old log /mysql/logs/error.log.85 does not exist
renaming /mysql/logs/error.log.84 to /mysql/logs/error.log.85 (rotatecount 120, logstart 1, i 84),
old log /mysql/logs/error.log.84 does not exist
renaming /mysql/logs/error.log.83 to /mysql/logs/error.log.84 (rotatecount 120, logstart 1, i 83),
old log /mysql/logs/error.log.83 does not exist
renaming /mysql/logs/error.log.82 to /mysql/logs/error.log.83 (rotatecount 120, logstart 1, i 82),
old log /mysql/logs/error.log.82 does not exist
renaming /mysql/logs/error.log.81 to /mysql/logs/error.log.82 (rotatecount 120, logstart 1, i 81),
old log /mysql/logs/error.log.81 does not exist
renaming /mysql/logs/error.log.80 to /mysql/logs/error.log.81 (rotatecount 120, logstart 1, i 80),
old log /mysql/logs/error.log.80 does not exist
renaming /mysql/logs/error.log.79 to /mysql/logs/error.log.80 (rotatecount 120, logstart 1, i 79),
old log /mysql/logs/error.log.79 does not exist
renaming /mysql/logs/error.log.78 to /mysql/logs/error.log.79 (rotatecount 120, logstart 1, i 78),
old log /mysql/logs/error.log.78 does not exist
renaming /mysql/logs/error.log.77 to /mysql/logs/error.log.78 (rotatecount 120, logstart 1, i 77),
old log /mysql/logs/error.log.77 does not exist
renaming /mysql/logs/error.log.76 to /mysql/logs/error.log.77 (rotatecount 120, logstart 1, i 76),
old log /mysql/logs/error.log.76 does not exist
renaming /mysql/logs/error.log.75 to /mysql/logs/error.log.76 (rotatecount 120, logstart 1, i 75),
old log /mysql/logs/error.log.75 does not exist
renaming /mysql/logs/error.log.74 to /mysql/logs/error.log.75 (rotatecount 120, logstart 1, i 74),
old log /mysql/logs/error.log.74 does not exist
renaming /mysql/logs/error.log.73 to /mysql/logs/error.log.74 (rotatecount 120, logstart 1, i 73),
old log /mysql/logs/error.log.73 does not exist
renaming /mysql/logs/error.log.72 to /mysql/logs/error.log.73 (rotatecount 120, logstart 1, i 72),
old log /mysql/logs/error.log.72 does not exist
renaming /mysql/logs/error.log.71 to /mysql/logs/error.log.72 (rotatecount 120, logstart 1, i 71),
old log /mysql/logs/error.log.71 does not exist
renaming /mysql/logs/error.log.70 to /mysql/logs/error.log.71 (rotatecount 120, logstart 1, i 70),
old log /mysql/logs/error.log.70 does not exist
renaming /mysql/logs/error.log.69 to /mysql/logs/error.log.70 (rotatecount 120, logstart 1, i 69),
old log /mysql/logs/error.log.69 does not exist
renaming /mysql/logs/error.log.68 to /mysql/logs/error.log.69 (rotatecount 120, logstart 1, i 68),
old log /mysql/logs/error.log.68 does not exist
renaming /mysql/logs/error.log.67 to /mysql/logs/error.log.68 (rotatecount 120, logstart 1, i 67),
old log /mysql/logs/error.log.67 does not exist
renaming /mysql/logs/error.log.66 to /mysql/logs/error.log.67 (rotatecount 120, logstart 1, i 66),
old log /mysql/logs/error.log.66 does not exist
renaming /mysql/logs/error.log.65 to /mysql/logs/error.log.66 (rotatecount 120, logstart 1, i 65),
old log /mysql/logs/error.log.65 does not exist
renaming /mysql/logs/error.log.64 to /mysql/logs/error.log.65 (rotatecount 120, logstart 1, i 64),
old log /mysql/logs/error.log.64 does not exist
renaming /mysql/logs/error.log.63 to /mysql/logs/error.log.64 (rotatecount 120, logstart 1, i 63),
old log /mysql/logs/error.log.63 does not exist
renaming /mysql/logs/error.log.62 to /mysql/logs/error.log.63 (rotatecount 120, logstart 1, i 62),
old log /mysql/logs/error.log.62 does not exist
renaming /mysql/logs/error.log.61 to /mysql/logs/error.log.62 (rotatecount 120, logstart 1, i 61),
old log /mysql/logs/error.log.61 does not exist
renaming /mysql/logs/error.log.60 to /mysql/logs/error.log.61 (rotatecount 120, logstart 1, i 60),
old log /mysql/logs/error.log.60 does not exist
renaming /mysql/logs/error.log.59 to /mysql/logs/error.log.60 (rotatecount 120, logstart 1, i 59),
old log /mysql/logs/error.log.59 does not exist
renaming /mysql/logs/error.log.58 to /mysql/logs/error.log.59 (rotatecount 120, logstart 1, i 58),
old log /mysql/logs/error.log.58 does not exist
renaming /mysql/logs/error.log.57 to /mysql/logs/error.log.58 (rotatecount 120, logstart 1, i 57),
old log /mysql/logs/error.log.57 does not exist
renaming /mysql/logs/error.log.56 to /mysql/logs/error.log.57 (rotatecount 120, logstart 1, i 56),
old log /mysql/logs/error.log.56 does not exist
renaming /mysql/logs/error.log.55 to /mysql/logs/error.log.56 (rotatecount 120, logstart 1, i 55),
old log /mysql/logs/error.log.55 does not exist
renaming /mysql/logs/error.log.54 to /mysql/logs/error.log.55 (rotatecount 120, logstart 1, i 54),
old log /mysql/logs/error.log.54 does not exist
renaming /mysql/logs/error.log.53 to /mysql/logs/error.log.54 (rotatecount 120, logstart 1, i 53),
old log /mysql/logs/error.log.53 does not exist
renaming /mysql/logs/error.log.52 to /mysql/logs/error.log.53 (rotatecount 120, logstart 1, i 52),
old log /mysql/logs/error.log.52 does not exist
renaming /mysql/logs/error.log.51 to /mysql/logs/error.log.52 (rotatecount 120, logstart 1, i 51),
old log /mysql/logs/error.log.51 does not exist
renaming /mysql/logs/error.log.50 to /mysql/logs/error.log.51 (rotatecount 120, logstart 1, i 50),
old log /mysql/logs/error.log.50 does not exist
renaming /mysql/logs/error.log.49 to /mysql/logs/error.log.50 (rotatecount 120, logstart 1, i 49),
old log /mysql/logs/error.log.49 does not exist
renaming /mysql/logs/error.log.48 to /mysql/logs/error.log.49 (rotatecount 120, logstart 1, i 48),
old log /mysql/logs/error.log.48 does not exist
renaming /mysql/logs/error.log.47 to /mysql/logs/error.log.48 (rotatecount 120, logstart 1, i 47),
old log /mysql/logs/error.log.47 does not exist
renaming /mysql/logs/error.log.46 to /mysql/logs/error.log.47 (rotatecount 120, logstart 1, i 46),
old log /mysql/logs/error.log.46 does not exist
renaming /mysql/logs/error.log.45 to /mysql/logs/error.log.46 (rotatecount 120, logstart 1, i 45),
old log /mysql/logs/error.log.45 does not exist
renaming /mysql/logs/error.log.44 to /mysql/logs/error.log.45 (rotatecount 120, logstart 1, i 44),
old log /mysql/logs/error.log.44 does not exist
renaming /mysql/logs/error.log.43 to /mysql/logs/error.log.44 (rotatecount 120, logstart 1, i 43),
old log /mysql/logs/error.log.43 does not exist
renaming /mysql/logs/error.log.42 to /mysql/logs/error.log.43 (rotatecount 120, logstart 1, i 42),
old log /mysql/logs/error.log.42 does not exist
renaming /mysql/logs/error.log.41 to /mysql/logs/error.log.42 (rotatecount 120, logstart 1, i 41),
old log /mysql/logs/error.log.41 does not exist
renaming /mysql/logs/error.log.40 to /mysql/logs/error.log.41 (rotatecount 120, logstart 1, i 40),
old log /mysql/logs/error.log.40 does not exist
renaming /mysql/logs/error.log.39 to /mysql/logs/error.log.40 (rotatecount 120, logstart 1, i 39),
old log /mysql/logs/error.log.39 does not exist
renaming /mysql/logs/error.log.38 to /mysql/logs/error.log.39 (rotatecount 120, logstart 1, i 38),
old log /mysql/logs/error.log.38 does not exist
renaming /mysql/logs/error.log.37 to /mysql/logs/error.log.38 (rotatecount 120, logstart 1, i 37),
old log /mysql/logs/error.log.37 does not exist
renaming /mysql/logs/error.log.36 to /mysql/logs/error.log.37 (rotatecount 120, logstart 1, i 36),
old log /mysql/logs/error.log.36 does not exist
renaming /mysql/logs/error.log.35 to /mysql/logs/error.log.36 (rotatecount 120, logstart 1, i 35),
old log /mysql/logs/error.log.35 does not exist
renaming /mysql/logs/error.log.34 to /mysql/logs/error.log.35 (rotatecount 120, logstart 1, i 34),
old log /mysql/logs/error.log.34 does not exist
renaming /mysql/logs/error.log.33 to /mysql/logs/error.log.34 (rotatecount 120, logstart 1, i 33),
old log /mysql/logs/error.log.33 does not exist
renaming /mysql/logs/error.log.32 to /mysql/logs/error.log.33 (rotatecount 120, logstart 1, i 32),
old log /mysql/logs/error.log.32 does not exist
renaming /mysql/logs/error.log.31 to /mysql/logs/error.log.32 (rotatecount 120, logstart 1, i 31),
old log /mysql/logs/error.log.31 does not exist
renaming /mysql/logs/error.log.30 to /mysql/logs/error.log.31 (rotatecount 120, logstart 1, i 30),
old log /mysql/logs/error.log.30 does not exist
renaming /mysql/logs/error.log.29 to /mysql/logs/error.log.30 (rotatecount 120, logstart 1, i 29),
old log /mysql/logs/error.log.29 does not exist
renaming /mysql/logs/error.log.28 to /mysql/logs/error.log.29 (rotatecount 120, logstart 1, i 28),
old log /mysql/logs/error.log.28 does not exist
renaming /mysql/logs/error.log.27 to /mysql/logs/error.log.28 (rotatecount 120, logstart 1, i 27),
old log /mysql/logs/error.log.27 does not exist
renaming /mysql/logs/error.log.26 to /mysql/logs/error.log.27 (rotatecount 120, logstart 1, i 26),
old log /mysql/logs/error.log.26 does not exist
renaming /mysql/logs/error.log.25 to /mysql/logs/error.log.26 (rotatecount 120, logstart 1, i 25),
old log /mysql/logs/error.log.25 does not exist
renaming /mysql/logs/error.log.24 to /mysql/logs/error.log.25 (rotatecount 120, logstart 1, i 24),
old log /mysql/logs/error.log.24 does not exist
renaming /mysql/logs/error.log.23 to /mysql/logs/error.log.24 (rotatecount 120, logstart 1, i 23),
old log /mysql/logs/error.log.23 does not exist
renaming /mysql/logs/error.log.22 to /mysql/logs/error.log.23 (rotatecount 120, logstart 1, i 22),
old log /mysql/logs/error.log.22 does not exist
renaming /mysql/logs/error.log.21 to /mysql/logs/error.log.22 (rotatecount 120, logstart 1, i 21),
old log /mysql/logs/error.log.21 does not exist
renaming /mysql/logs/error.log.20 to /mysql/logs/error.log.21 (rotatecount 120, logstart 1, i 20),
old log /mysql/logs/error.log.20 does not exist
renaming /mysql/logs/error.log.19 to /mysql/logs/error.log.20 (rotatecount 120, logstart 1, i 19),
old log /mysql/logs/error.log.19 does not exist
renaming /mysql/logs/error.log.18 to /mysql/logs/error.log.19 (rotatecount 120, logstart 1, i 18),
old log /mysql/logs/error.log.18 does not exist
renaming /mysql/logs/error.log.17 to /mysql/logs/error.log.18 (rotatecount 120, logstart 1, i 17),
old log /mysql/logs/error.log.17 does not exist
renaming /mysql/logs/error.log.16 to /mysql/logs/error.log.17 (rotatecount 120, logstart 1, i 16),
old log /mysql/logs/error.log.16 does not exist
renaming /mysql/logs/error.log.15 to /mysql/logs/error.log.16 (rotatecount 120, logstart 1, i 15),
old log /mysql/logs/error.log.15 does not exist
renaming /mysql/logs/error.log.14 to /mysql/logs/error.log.15 (rotatecount 120, logstart 1, i 14),
old log /mysql/logs/error.log.14 does not exist
renaming /mysql/logs/error.log.13 to /mysql/logs/error.log.14 (rotatecount 120, logstart 1, i 13),
old log /mysql/logs/error.log.13 does not exist
renaming /mysql/logs/error.log.12 to /mysql/logs/error.log.13 (rotatecount 120, logstart 1, i 12),
old log /mysql/logs/error.log.12 does not exist
renaming /mysql/logs/error.log.11 to /mysql/logs/error.log.12 (rotatecount 120, logstart 1, i 11),
old log /mysql/logs/error.log.11 does not exist
renaming /mysql/logs/error.log.10 to /mysql/logs/error.log.11 (rotatecount 120, logstart 1, i 10),
old log /mysql/logs/error.log.10 does not exist
renaming /mysql/logs/error.log.9 to /mysql/logs/error.log.10 (rotatecount 120, logstart 1, i 9),
old log /mysql/logs/error.log.9 does not exist
renaming /mysql/logs/error.log.8 to /mysql/logs/error.log.9 (rotatecount 120, logstart 1, i 8),
old log /mysql/logs/error.log.8 does not exist
renaming /mysql/logs/error.log.7 to /mysql/logs/error.log.8 (rotatecount 120, logstart 1, i 7),
old log /mysql/logs/error.log.7 does not exist
renaming /mysql/logs/error.log.6 to /mysql/logs/error.log.7 (rotatecount 120, logstart 1, i 6),
old log /mysql/logs/error.log.6 does not exist
renaming /mysql/logs/error.log.5 to /mysql/logs/error.log.6 (rotatecount 120, logstart 1, i 5),
old log /mysql/logs/error.log.5 does not exist
renaming /mysql/logs/error.log.4 to /mysql/logs/error.log.5 (rotatecount 120, logstart 1, i 4),
old log /mysql/logs/error.log.4 does not exist
renaming /mysql/logs/error.log.3 to /mysql/logs/error.log.4 (rotatecount 120, logstart 1, i 3),
old log /mysql/logs/error.log.3 does not exist
renaming /mysql/logs/error.log.2 to /mysql/logs/error.log.3 (rotatecount 120, logstart 1, i 2),
renaming /mysql/logs/error.log.1 to /mysql/logs/error.log.2 (rotatecount 120, logstart 1, i 1),
renaming /mysql/logs/error.log.0 to /mysql/logs/error.log.1 (rotatecount 120, logstart 1, i 0),
old log /mysql/logs/error.log.0 does not exist
log /mysql/logs/error.log.121 doesn't exist -- won't try to dispose of it
rotating log /mysql/logs/general.log, log->rotateCount is 120
renaming /mysql/logs/general.log.120 to /mysql/logs/general.log.121 (rotatecount 120, logstart 1, i 120),
old log /mysql/logs/general.log.120 does not exist
renaming /mysql/logs/general.log.119 to /mysql/logs/general.log.120 (rotatecount 120, logstart 1, i 119),
old log /mysql/logs/general.log.119 does not exist
renaming /mysql/logs/general.log.118 to /mysql/logs/general.log.119 (rotatecount 120, logstart 1, i 118),
old log /mysql/logs/general.log.118 does not exist
renaming /mysql/logs/general.log.117 to /mysql/logs/general.log.118 (rotatecount 120, logstart 1, i 117),
old log /mysql/logs/general.log.117 does not exist
renaming /mysql/logs/general.log.116 to /mysql/logs/general.log.117 (rotatecount 120, logstart 1, i 116),
old log /mysql/logs/general.log.116 does not exist
renaming /mysql/logs/general.log.115 to /mysql/logs/general.log.116 (rotatecount 120, logstart 1, i 115),
old log /mysql/logs/general.log.115 does not exist
renaming /mysql/logs/general.log.114 to /mysql/logs/general.log.115 (rotatecount 120, logstart 1, i 114),
old log /mysql/logs/general.log.114 does not exist
renaming /mysql/logs/general.log.113 to /mysql/logs/general.log.114 (rotatecount 120, logstart 1, i 113),
old log /mysql/logs/general.log.113 does not exist
renaming /mysql/logs/general.log.112 to /mysql/logs/general.log.113 (rotatecount 120, logstart 1, i 112),
old log /mysql/logs/general.log.112 does not exist
renaming /mysql/logs/general.log.111 to /mysql/logs/general.log.112 (rotatecount 120, logstart 1, i 111),
old log /mysql/logs/general.log.111 does not exist
renaming /mysql/logs/general.log.110 to /mysql/logs/general.log.111 (rotatecount 120, logstart 1, i 110),
old log /mysql/logs/general.log.110 does not exist
renaming /mysql/logs/general.log.109 to /mysql/logs/general.log.110 (rotatecount 120, logstart 1, i 109),
old log /mysql/logs/general.log.109 does not exist
renaming /mysql/logs/general.log.108 to /mysql/logs/general.log.109 (rotatecount 120, logstart 1, i 108),
old log /mysql/logs/general.log.108 does not exist
renaming /mysql/logs/general.log.107 to /mysql/logs/general.log.108 (rotatecount 120, logstart 1, i 107),
old log /mysql/logs/general.log.107 does not exist
renaming /mysql/logs/general.log.106 to /mysql/logs/general.log.107 (rotatecount 120, logstart 1, i 106),
old log /mysql/logs/general.log.106 does not exist
renaming /mysql/logs/general.log.105 to /mysql/logs/general.log.106 (rotatecount 120, logstart 1, i 105),
old log /mysql/logs/general.log.105 does not exist
renaming /mysql/logs/general.log.104 to /mysql/logs/general.log.105 (rotatecount 120, logstart 1, i 104),
old log /mysql/logs/general.log.104 does not exist
renaming /mysql/logs/general.log.103 to /mysql/logs/general.log.104 (rotatecount 120, logstart 1, i 103),
old log /mysql/logs/general.log.103 does not exist
renaming /mysql/logs/general.log.102 to /mysql/logs/general.log.103 (rotatecount 120, logstart 1, i 102),
old log /mysql/logs/general.log.102 does not exist
renaming /mysql/logs/general.log.101 to /mysql/logs/general.log.102 (rotatecount 120, logstart 1, i 101),
old log /mysql/logs/general.log.101 does not exist
renaming /mysql/logs/general.log.100 to /mysql/logs/general.log.101 (rotatecount 120, logstart 1, i 100),
old log /mysql/logs/general.log.100 does not exist
renaming /mysql/logs/general.log.99 to /mysql/logs/general.log.100 (rotatecount 120, logstart 1, i 99),
old log /mysql/logs/general.log.99 does not exist
renaming /mysql/logs/general.log.98 to /mysql/logs/general.log.99 (rotatecount 120, logstart 1, i 98),
old log /mysql/logs/general.log.98 does not exist
renaming /mysql/logs/general.log.97 to /mysql/logs/general.log.98 (rotatecount 120, logstart 1, i 97),
old log /mysql/logs/general.log.97 does not exist
renaming /mysql/logs/general.log.96 to /mysql/logs/general.log.97 (rotatecount 120, logstart 1, i 96),
old log /mysql/logs/general.log.96 does not exist
renaming /mysql/logs/general.log.95 to /mysql/logs/general.log.96 (rotatecount 120, logstart 1, i 95),
old log /mysql/logs/general.log.95 does not exist
renaming /mysql/logs/general.log.94 to /mysql/logs/general.log.95 (rotatecount 120, logstart 1, i 94),
old log /mysql/logs/general.log.94 does not exist
renaming /mysql/logs/general.log.93 to /mysql/logs/general.log.94 (rotatecount 120, logstart 1, i 93),
old log /mysql/logs/general.log.93 does not exist
renaming /mysql/logs/general.log.92 to /mysql/logs/general.log.93 (rotatecount 120, logstart 1, i 92),
old log /mysql/logs/general.log.92 does not exist
renaming /mysql/logs/general.log.91 to /mysql/logs/general.log.92 (rotatecount 120, logstart 1, i 91),
old log /mysql/logs/general.log.91 does not exist
renaming /mysql/logs/general.log.90 to /mysql/logs/general.log.91 (rotatecount 120, logstart 1, i 90),
old log /mysql/logs/general.log.90 does not exist
renaming /mysql/logs/general.log.89 to /mysql/logs/general.log.90 (rotatecount 120, logstart 1, i 89),
old log /mysql/logs/general.log.89 does not exist
renaming /mysql/logs/general.log.88 to /mysql/logs/general.log.89 (rotatecount 120, logstart 1, i 88),
old log /mysql/logs/general.log.88 does not exist
renaming /mysql/logs/general.log.87 to /mysql/logs/general.log.88 (rotatecount 120, logstart 1, i 87),
old log /mysql/logs/general.log.87 does not exist
renaming /mysql/logs/general.log.86 to /mysql/logs/general.log.87 (rotatecount 120, logstart 1, i 86),
old log /mysql/logs/general.log.86 does not exist
renaming /mysql/logs/general.log.85 to /mysql/logs/general.log.86 (rotatecount 120, logstart 1, i 85),
old log /mysql/logs/general.log.85 does not exist
renaming /mysql/logs/general.log.84 to /mysql/logs/general.log.85 (rotatecount 120, logstart 1, i 84),
old log /mysql/logs/general.log.84 does not exist
renaming /mysql/logs/general.log.83 to /mysql/logs/general.log.84 (rotatecount 120, logstart 1, i 83),
old log /mysql/logs/general.log.83 does not exist
renaming /mysql/logs/general.log.82 to /mysql/logs/general.log.83 (rotatecount 120, logstart 1, i 82),
old log /mysql/logs/general.log.82 does not exist
renaming /mysql/logs/general.log.81 to /mysql/logs/general.log.82 (rotatecount 120, logstart 1, i 81),
old log /mysql/logs/general.log.81 does not exist
renaming /mysql/logs/general.log.80 to /mysql/logs/general.log.81 (rotatecount 120, logstart 1, i 80),
old log /mysql/logs/general.log.80 does not exist
renaming /mysql/logs/general.log.79 to /mysql/logs/general.log.80 (rotatecount 120, logstart 1, i 79),
old log /mysql/logs/general.log.79 does not exist
renaming /mysql/logs/general.log.78 to /mysql/logs/general.log.79 (rotatecount 120, logstart 1, i 78),
old log /mysql/logs/general.log.78 does not exist
renaming /mysql/logs/general.log.77 to /mysql/logs/general.log.78 (rotatecount 120, logstart 1, i 77),
old log /mysql/logs/general.log.77 does not exist
renaming /mysql/logs/general.log.76 to /mysql/logs/general.log.77 (rotatecount 120, logstart 1, i 76),
old log /mysql/logs/general.log.76 does not exist
renaming /mysql/logs/general.log.75 to /mysql/logs/general.log.76 (rotatecount 120, logstart 1, i 75),
old log /mysql/logs/general.log.75 does not exist
renaming /mysql/logs/general.log.74 to /mysql/logs/general.log.75 (rotatecount 120, logstart 1, i 74),
old log /mysql/logs/general.log.74 does not exist
renaming /mysql/logs/general.log.73 to /mysql/logs/general.log.74 (rotatecount 120, logstart 1, i 73),
old log /mysql/logs/general.log.73 does not exist
renaming /mysql/logs/general.log.72 to /mysql/logs/general.log.73 (rotatecount 120, logstart 1, i 72),
old log /mysql/logs/general.log.72 does not exist
renaming /mysql/logs/general.log.71 to /mysql/logs/general.log.72 (rotatecount 120, logstart 1, i 71),
old log /mysql/logs/general.log.71 does not exist
renaming /mysql/logs/general.log.70 to /mysql/logs/general.log.71 (rotatecount 120, logstart 1, i 70),
old log /mysql/logs/general.log.70 does not exist
renaming /mysql/logs/general.log.69 to /mysql/logs/general.log.70 (rotatecount 120, logstart 1, i 69),
old log /mysql/logs/general.log.69 does not exist
renaming /mysql/logs/general.log.68 to /mysql/logs/general.log.69 (rotatecount 120, logstart 1, i 68),
old log /mysql/logs/general.log.68 does not exist
renaming /mysql/logs/general.log.67 to /mysql/logs/general.log.68 (rotatecount 120, logstart 1, i 67),
old log /mysql/logs/general.log.67 does not exist
renaming /mysql/logs/general.log.66 to /mysql/logs/general.log.67 (rotatecount 120, logstart 1, i 66),
old log /mysql/logs/general.log.66 does not exist
renaming /mysql/logs/general.log.65 to /mysql/logs/general.log.66 (rotatecount 120, logstart 1, i 65),
old log /mysql/logs/general.log.65 does not exist
renaming /mysql/logs/general.log.64 to /mysql/logs/general.log.65 (rotatecount 120, logstart 1, i 64),
old log /mysql/logs/general.log.64 does not exist
renaming /mysql/logs/general.log.63 to /mysql/logs/general.log.64 (rotatecount 120, logstart 1, i 63),
old log /mysql/logs/general.log.63 does not exist
renaming /mysql/logs/general.log.62 to /mysql/logs/general.log.63 (rotatecount 120, logstart 1, i 62),
old log /mysql/logs/general.log.62 does not exist
renaming /mysql/logs/general.log.61 to /mysql/logs/general.log.62 (rotatecount 120, logstart 1, i 61),
old log /mysql/logs/general.log.61 does not exist
renaming /mysql/logs/general.log.60 to /mysql/logs/general.log.61 (rotatecount 120, logstart 1, i 60),
old log /mysql/logs/general.log.60 does not exist
renaming /mysql/logs/general.log.59 to /mysql/logs/general.log.60 (rotatecount 120, logstart 1, i 59),
old log /mysql/logs/general.log.59 does not exist
renaming /mysql/logs/general.log.58 to /mysql/logs/general.log.59 (rotatecount 120, logstart 1, i 58),
old log /mysql/logs/general.log.58 does not exist
renaming /mysql/logs/general.log.57 to /mysql/logs/general.log.58 (rotatecount 120, logstart 1, i 57),
old log /mysql/logs/general.log.57 does not exist
renaming /mysql/logs/general.log.56 to /mysql/logs/general.log.57 (rotatecount 120, logstart 1, i 56),
old log /mysql/logs/general.log.56 does not exist
renaming /mysql/logs/general.log.55 to /mysql/logs/general.log.56 (rotatecount 120, logstart 1, i 55),
old log /mysql/logs/general.log.55 does not exist
renaming /mysql/logs/general.log.54 to /mysql/logs/general.log.55 (rotatecount 120, logstart 1, i 54),
old log /mysql/logs/general.log.54 does not exist
renaming /mysql/logs/general.log.53 to /mysql/logs/general.log.54 (rotatecount 120, logstart 1, i 53),
old log /mysql/logs/general.log.53 does not exist
renaming /mysql/logs/general.log.52 to /mysql/logs/general.log.53 (rotatecount 120, logstart 1, i 52),
old log /mysql/logs/general.log.52 does not exist
renaming /mysql/logs/general.log.51 to /mysql/logs/general.log.52 (rotatecount 120, logstart 1, i 51),
old log /mysql/logs/general.log.51 does not exist
renaming /mysql/logs/general.log.50 to /mysql/logs/general.log.51 (rotatecount 120, logstart 1, i 50),
old log /mysql/logs/general.log.50 does not exist
renaming /mysql/logs/general.log.49 to /mysql/logs/general.log.50 (rotatecount 120, logstart 1, i 49),
old log /mysql/logs/general.log.49 does not exist
renaming /mysql/logs/general.log.48 to /mysql/logs/general.log.49 (rotatecount 120, logstart 1, i 48),
old log /mysql/logs/general.log.48 does not exist
renaming /mysql/logs/general.log.47 to /mysql/logs/general.log.48 (rotatecount 120, logstart 1, i 47),
old log /mysql/logs/general.log.47 does not exist
renaming /mysql/logs/general.log.46 to /mysql/logs/general.log.47 (rotatecount 120, logstart 1, i 46),
old log /mysql/logs/general.log.46 does not exist
renaming /mysql/logs/general.log.45 to /mysql/logs/general.log.46 (rotatecount 120, logstart 1, i 45),
old log /mysql/logs/general.log.45 does not exist
renaming /mysql/logs/general.log.44 to /mysql/logs/general.log.45 (rotatecount 120, logstart 1, i 44),
old log /mysql/logs/general.log.44 does not exist
renaming /mysql/logs/general.log.43 to /mysql/logs/general.log.44 (rotatecount 120, logstart 1, i 43),
old log /mysql/logs/general.log.43 does not exist
renaming /mysql/logs/general.log.42 to /mysql/logs/general.log.43 (rotatecount 120, logstart 1, i 42),
old log /mysql/logs/general.log.42 does not exist
renaming /mysql/logs/general.log.41 to /mysql/logs/general.log.42 (rotatecount 120, logstart 1, i 41),
old log /mysql/logs/general.log.41 does not exist
renaming /mysql/logs/general.log.40 to /mysql/logs/general.log.41 (rotatecount 120, logstart 1, i 40),
old log /mysql/logs/general.log.40 does not exist
renaming /mysql/logs/general.log.39 to /mysql/logs/general.log.40 (rotatecount 120, logstart 1, i 39),
old log /mysql/logs/general.log.39 does not exist
renaming /mysql/logs/general.log.38 to /mysql/logs/general.log.39 (rotatecount 120, logstart 1, i 38),
old log /mysql/logs/general.log.38 does not exist
renaming /mysql/logs/general.log.37 to /mysql/logs/general.log.38 (rotatecount 120, logstart 1, i 37),
old log /mysql/logs/general.log.37 does not exist
renaming /mysql/logs/general.log.36 to /mysql/logs/general.log.37 (rotatecount 120, logstart 1, i 36),
old log /mysql/logs/general.log.36 does not exist
renaming /mysql/logs/general.log.35 to /mysql/logs/general.log.36 (rotatecount 120, logstart 1, i 35),
old log /mysql/logs/general.log.35 does not exist
renaming /mysql/logs/general.log.34 to /mysql/logs/general.log.35 (rotatecount 120, logstart 1, i 34),
old log /mysql/logs/general.log.34 does not exist
renaming /mysql/logs/general.log.33 to /mysql/logs/general.log.34 (rotatecount 120, logstart 1, i 33),
old log /mysql/logs/general.log.33 does not exist
renaming /mysql/logs/general.log.32 to /mysql/logs/general.log.33 (rotatecount 120, logstart 1, i 32),
old log /mysql/logs/general.log.32 does not exist
renaming /mysql/logs/general.log.31 to /mysql/logs/general.log.32 (rotatecount 120, logstart 1, i 31),
old log /mysql/logs/general.log.31 does not exist
renaming /mysql/logs/general.log.30 to /mysql/logs/general.log.31 (rotatecount 120, logstart 1, i 30),
old log /mysql/logs/general.log.30 does not exist
renaming /mysql/logs/general.log.29 to /mysql/logs/general.log.30 (rotatecount 120, logstart 1, i 29),
old log /mysql/logs/general.log.29 does not exist
renaming /mysql/logs/general.log.28 to /mysql/logs/general.log.29 (rotatecount 120, logstart 1, i 28),
old log /mysql/logs/general.log.28 does not exist
renaming /mysql/logs/general.log.27 to /mysql/logs/general.log.28 (rotatecount 120, logstart 1, i 27),
old log /mysql/logs/general.log.27 does not exist
renaming /mysql/logs/general.log.26 to /mysql/logs/general.log.27 (rotatecount 120, logstart 1, i 26),
old log /mysql/logs/general.log.26 does not exist
renaming /mysql/logs/general.log.25 to /mysql/logs/general.log.26 (rotatecount 120, logstart 1, i 25),
old log /mysql/logs/general.log.25 does not exist
renaming /mysql/logs/general.log.24 to /mysql/logs/general.log.25 (rotatecount 120, logstart 1, i 24),
old log /mysql/logs/general.log.24 does not exist
renaming /mysql/logs/general.log.23 to /mysql/logs/general.log.24 (rotatecount 120, logstart 1, i 23),
old log /mysql/logs/general.log.23 does not exist
renaming /mysql/logs/general.log.22 to /mysql/logs/general.log.23 (rotatecount 120, logstart 1, i 22),
old log /mysql/logs/general.log.22 does not exist
renaming /mysql/logs/general.log.21 to /mysql/logs/general.log.22 (rotatecount 120, logstart 1, i 21),
old log /mysql/logs/general.log.21 does not exist
renaming /mysql/logs/general.log.20 to /mysql/logs/general.log.21 (rotatecount 120, logstart 1, i 20),
old log /mysql/logs/general.log.20 does not exist
renaming /mysql/logs/general.log.19 to /mysql/logs/general.log.20 (rotatecount 120, logstart 1, i 19),
old log /mysql/logs/general.log.19 does not exist
renaming /mysql/logs/general.log.18 to /mysql/logs/general.log.19 (rotatecount 120, logstart 1, i 18),
old log /mysql/logs/general.log.18 does not exist
renaming /mysql/logs/general.log.17 to /mysql/logs/general.log.18 (rotatecount 120, logstart 1, i 17),
old log /mysql/logs/general.log.17 does not exist
renaming /mysql/logs/general.log.16 to /mysql/logs/general.log.17 (rotatecount 120, logstart 1, i 16),
old log /mysql/logs/general.log.16 does not exist
renaming /mysql/logs/general.log.15 to /mysql/logs/general.log.16 (rotatecount 120, logstart 1, i 15),
old log /mysql/logs/general.log.15 does not exist
renaming /mysql/logs/general.log.14 to /mysql/logs/general.log.15 (rotatecount 120, logstart 1, i 14),
old log /mysql/logs/general.log.14 does not exist
renaming /mysql/logs/general.log.13 to /mysql/logs/general.log.14 (rotatecount 120, logstart 1, i 13),
old log /mysql/logs/general.log.13 does not exist
renaming /mysql/logs/general.log.12 to /mysql/logs/general.log.13 (rotatecount 120, logstart 1, i 12),
old log /mysql/logs/general.log.12 does not exist
renaming /mysql/logs/general.log.11 to /mysql/logs/general.log.12 (rotatecount 120, logstart 1, i 11),
old log /mysql/logs/general.log.11 does not exist
renaming /mysql/logs/general.log.10 to /mysql/logs/general.log.11 (rotatecount 120, logstart 1, i 10),
old log /mysql/logs/general.log.10 does not exist
renaming /mysql/logs/general.log.9 to /mysql/logs/general.log.10 (rotatecount 120, logstart 1, i 9),
old log /mysql/logs/general.log.9 does not exist
renaming /mysql/logs/general.log.8 to /mysql/logs/general.log.9 (rotatecount 120, logstart 1, i 8),
old log /mysql/logs/general.log.8 does not exist
renaming /mysql/logs/general.log.7 to /mysql/logs/general.log.8 (rotatecount 120, logstart 1, i 7),
old log /mysql/logs/general.log.7 does not exist
renaming /mysql/logs/general.log.6 to /mysql/logs/general.log.7 (rotatecount 120, logstart 1, i 6),
old log /mysql/logs/general.log.6 does not exist
renaming /mysql/logs/general.log.5 to /mysql/logs/general.log.6 (rotatecount 120, logstart 1, i 5),
old log /mysql/logs/general.log.5 does not exist
renaming /mysql/logs/general.log.4 to /mysql/logs/general.log.5 (rotatecount 120, logstart 1, i 4),
old log /mysql/logs/general.log.4 does not exist
renaming /mysql/logs/general.log.3 to /mysql/logs/general.log.4 (rotatecount 120, logstart 1, i 3),
renaming /mysql/logs/general.log.2 to /mysql/logs/general.log.3 (rotatecount 120, logstart 1, i 2),
renaming /mysql/logs/general.log.1 to /mysql/logs/general.log.2 (rotatecount 120, logstart 1, i 1),
renaming /mysql/logs/general.log.0 to /mysql/logs/general.log.1 (rotatecount 120, logstart 1, i 0),
old log /mysql/logs/general.log.0 does not exist
log /mysql/logs/general.log.121 doesn't exist -- won't try to dispose of it
rotating log /mysql/logs/slow-queries.log, log->rotateCount is 120
renaming /mysql/logs/slow-queries.log.120 to /mysql/logs/slow-queries.log.121 (rotatecount 120, logstart 1, i 120),
old log /mysql/logs/slow-queries.log.120 does not exist
renaming /mysql/logs/slow-queries.log.119 to /mysql/logs/slow-queries.log.120 (rotatecount 120, logstart 1, i 119),
old log /mysql/logs/slow-queries.log.119 does not exist
renaming /mysql/logs/slow-queries.log.118 to /mysql/logs/slow-queries.log.119 (rotatecount 120, logstart 1, i 118),
old log /mysql/logs/slow-queries.log.118 does not exist
renaming /mysql/logs/slow-queries.log.117 to /mysql/logs/slow-queries.log.118 (rotatecount 120, logstart 1, i 117),
old log /mysql/logs/slow-queries.log.117 does not exist
renaming /mysql/logs/slow-queries.log.116 to /mysql/logs/slow-queries.log.117 (rotatecount 120, logstart 1, i 116),
old log /mysql/logs/slow-queries.log.116 does not exist
renaming /mysql/logs/slow-queries.log.115 to /mysql/logs/slow-queries.log.116 (rotatecount 120, logstart 1, i 115),
old log /mysql/logs/slow-queries.log.115 does not exist
renaming /mysql/logs/slow-queries.log.114 to /mysql/logs/slow-queries.log.115 (rotatecount 120, logstart 1, i 114),
old log /mysql/logs/slow-queries.log.114 does not exist
renaming /mysql/logs/slow-queries.log.113 to /mysql/logs/slow-queries.log.114 (rotatecount 120, logstart 1, i 113),
old log /mysql/logs/slow-queries.log.113 does not exist
renaming /mysql/logs/slow-queries.log.112 to /mysql/logs/slow-queries.log.113 (rotatecount 120, logstart 1, i 112),
old log /mysql/logs/slow-queries.log.112 does not exist
renaming /mysql/logs/slow-queries.log.111 to /mysql/logs/slow-queries.log.112 (rotatecount 120, logstart 1, i 111),
old log /mysql/logs/slow-queries.log.111 does not exist
renaming /mysql/logs/slow-queries.log.110 to /mysql/logs/slow-queries.log.111 (rotatecount 120, logstart 1, i 110),
old log /mysql/logs/slow-queries.log.110 does not exist
renaming /mysql/logs/slow-queries.log.109 to /mysql/logs/slow-queries.log.110 (rotatecount 120, logstart 1, i 109),
old log /mysql/logs/slow-queries.log.109 does not exist
renaming /mysql/logs/slow-queries.log.108 to /mysql/logs/slow-queries.log.109 (rotatecount 120, logstart 1, i 108),
old log /mysql/logs/slow-queries.log.108 does not exist
renaming /mysql/logs/slow-queries.log.107 to /mysql/logs/slow-queries.log.108 (rotatecount 120, logstart 1, i 107),
old log /mysql/logs/slow-queries.log.107 does not exist
renaming /mysql/logs/slow-queries.log.106 to /mysql/logs/slow-queries.log.107 (rotatecount 120, logstart 1, i 106),
old log /mysql/logs/slow-queries.log.106 does not exist
renaming /mysql/logs/slow-queries.log.105 to /mysql/logs/slow-queries.log.106 (rotatecount 120, logstart 1, i 105),
old log /mysql/logs/slow-queries.log.105 does not exist
renaming /mysql/logs/slow-queries.log.104 to /mysql/logs/slow-queries.log.105 (rotatecount 120, logstart 1, i 104),
old log /mysql/logs/slow-queries.log.104 does not exist
renaming /mysql/logs/slow-queries.log.103 to /mysql/logs/slow-queries.log.104 (rotatecount 120, logstart 1, i 103),
old log /mysql/logs/slow-queries.log.103 does not exist
renaming /mysql/logs/slow-queries.log.102 to /mysql/logs/slow-queries.log.103 (rotatecount 120, logstart 1, i 102),
old log /mysql/logs/slow-queries.log.102 does not exist
renaming /mysql/logs/slow-queries.log.101 to /mysql/logs/slow-queries.log.102 (rotatecount 120, logstart 1, i 101),
old log /mysql/logs/slow-queries.log.101 does not exist
renaming /mysql/logs/slow-queries.log.100 to /mysql/logs/slow-queries.log.101 (rotatecount 120, logstart 1, i 100),
old log /mysql/logs/slow-queries.log.100 does not exist
renaming /mysql/logs/slow-queries.log.99 to /mysql/logs/slow-queries.log.100 (rotatecount 120, logstart 1, i 99),
old log /mysql/logs/slow-queries.log.99 does not exist
renaming /mysql/logs/slow-queries.log.98 to /mysql/logs/slow-queries.log.99 (rotatecount 120, logstart 1, i 98),
old log /mysql/logs/slow-queries.log.98 does not exist
renaming /mysql/logs/slow-queries.log.97 to /mysql/logs/slow-queries.log.98 (rotatecount 120, logstart 1, i 97),
old log /mysql/logs/slow-queries.log.97 does not exist
renaming /mysql/logs/slow-queries.log.96 to /mysql/logs/slow-queries.log.97 (rotatecount 120, logstart 1, i 96),
old log /mysql/logs/slow-queries.log.96 does not exist
renaming /mysql/logs/slow-queries.log.95 to /mysql/logs/slow-queries.log.96 (rotatecount 120, logstart 1, i 95),
old log /mysql/logs/slow-queries.log.95 does not exist
renaming /mysql/logs/slow-queries.log.94 to /mysql/logs/slow-queries.log.95 (rotatecount 120, logstart 1, i 94),
old log /mysql/logs/slow-queries.log.94 does not exist
renaming /mysql/logs/slow-queries.log.93 to /mysql/logs/slow-queries.log.94 (rotatecount 120, logstart 1, i 93),
old log /mysql/logs/slow-queries.log.93 does not exist
renaming /mysql/logs/slow-queries.log.92 to /mysql/logs/slow-queries.log.93 (rotatecount 120, logstart 1, i 92),
old log /mysql/logs/slow-queries.log.92 does not exist
renaming /mysql/logs/slow-queries.log.91 to /mysql/logs/slow-queries.log.92 (rotatecount 120, logstart 1, i 91),
old log /mysql/logs/slow-queries.log.91 does not exist
renaming /mysql/logs/slow-queries.log.90 to /mysql/logs/slow-queries.log.91 (rotatecount 120, logstart 1, i 90),
old log /mysql/logs/slow-queries.log.90 does not exist
renaming /mysql/logs/slow-queries.log.89 to /mysql/logs/slow-queries.log.90 (rotatecount 120, logstart 1, i 89),
old log /mysql/logs/slow-queries.log.89 does not exist
renaming /mysql/logs/slow-queries.log.88 to /mysql/logs/slow-queries.log.89 (rotatecount 120, logstart 1, i 88),
old log /mysql/logs/slow-queries.log.88 does not exist
renaming /mysql/logs/slow-queries.log.87 to /mysql/logs/slow-queries.log.88 (rotatecount 120, logstart 1, i 87),
old log /mysql/logs/slow-queries.log.87 does not exist
renaming /mysql/logs/slow-queries.log.86 to /mysql/logs/slow-queries.log.87 (rotatecount 120, logstart 1, i 86),
old log /mysql/logs/slow-queries.log.86 does not exist
renaming /mysql/logs/slow-queries.log.85 to /mysql/logs/slow-queries.log.86 (rotatecount 120, logstart 1, i 85),
old log /mysql/logs/slow-queries.log.85 does not exist
renaming /mysql/logs/slow-queries.log.84 to /mysql/logs/slow-queries.log.85 (rotatecount 120, logstart 1, i 84),
old log /mysql/logs/slow-queries.log.84 does not exist
renaming /mysql/logs/slow-queries.log.83 to /mysql/logs/slow-queries.log.84 (rotatecount 120, logstart 1, i 83),
old log /mysql/logs/slow-queries.log.83 does not exist
renaming /mysql/logs/slow-queries.log.82 to /mysql/logs/slow-queries.log.83 (rotatecount 120, logstart 1, i 82),
old log /mysql/logs/slow-queries.log.82 does not exist
renaming /mysql/logs/slow-queries.log.81 to /mysql/logs/slow-queries.log.82 (rotatecount 120, logstart 1, i 81),
old log /mysql/logs/slow-queries.log.81 does not exist
renaming /mysql/logs/slow-queries.log.80 to /mysql/logs/slow-queries.log.81 (rotatecount 120, logstart 1, i 80),
old log /mysql/logs/slow-queries.log.80 does not exist
renaming /mysql/logs/slow-queries.log.79 to /mysql/logs/slow-queries.log.80 (rotatecount 120, logstart 1, i 79),
old log /mysql/logs/slow-queries.log.79 does not exist
renaming /mysql/logs/slow-queries.log.78 to /mysql/logs/slow-queries.log.79 (rotatecount 120, logstart 1, i 78),
old log /mysql/logs/slow-queries.log.78 does not exist
renaming /mysql/logs/slow-queries.log.77 to /mysql/logs/slow-queries.log.78 (rotatecount 120, logstart 1, i 77),
old log /mysql/logs/slow-queries.log.77 does not exist
renaming /mysql/logs/slow-queries.log.76 to /mysql/logs/slow-queries.log.77 (rotatecount 120, logstart 1, i 76),
old log /mysql/logs/slow-queries.log.76 does not exist
renaming /mysql/logs/slow-queries.log.75 to /mysql/logs/slow-queries.log.76 (rotatecount 120, logstart 1, i 75),
old log /mysql/logs/slow-queries.log.75 does not exist
renaming /mysql/logs/slow-queries.log.74 to /mysql/logs/slow-queries.log.75 (rotatecount 120, logstart 1, i 74),
old log /mysql/logs/slow-queries.log.74 does not exist
renaming /mysql/logs/slow-queries.log.73 to /mysql/logs/slow-queries.log.74 (rotatecount 120, logstart 1, i 73),
old log /mysql/logs/slow-queries.log.73 does not exist
renaming /mysql/logs/slow-queries.log.72 to /mysql/logs/slow-queries.log.73 (rotatecount 120, logstart 1, i 72),
old log /mysql/logs/slow-queries.log.72 does not exist
renaming /mysql/logs/slow-queries.log.71 to /mysql/logs/slow-queries.log.72 (rotatecount 120, logstart 1, i 71),
old log /mysql/logs/slow-queries.log.71 does not exist
renaming /mysql/logs/slow-queries.log.70 to /mysql/logs/slow-queries.log.71 (rotatecount 120, logstart 1, i 70),
old log /mysql/logs/slow-queries.log.70 does not exist
renaming /mysql/logs/slow-queries.log.69 to /mysql/logs/slow-queries.log.70 (rotatecount 120, logstart 1, i 69),
old log /mysql/logs/slow-queries.log.69 does not exist
renaming /mysql/logs/slow-queries.log.68 to /mysql/logs/slow-queries.log.69 (rotatecount 120, logstart 1, i 68),
old log /mysql/logs/slow-queries.log.68 does not exist
renaming /mysql/logs/slow-queries.log.67 to /mysql/logs/slow-queries.log.68 (rotatecount 120, logstart 1, i 67),
old log /mysql/logs/slow-queries.log.67 does not exist
renaming /mysql/logs/slow-queries.log.66 to /mysql/logs/slow-queries.log.67 (rotatecount 120, logstart 1, i 66),
old log /mysql/logs/slow-queries.log.66 does not exist
renaming /mysql/logs/slow-queries.log.65 to /mysql/logs/slow-queries.log.66 (rotatecount 120, logstart 1, i 65),
old log /mysql/logs/slow-queries.log.65 does not exist
renaming /mysql/logs/slow-queries.log.64 to /mysql/logs/slow-queries.log.65 (rotatecount 120, logstart 1, i 64),
old log /mysql/logs/slow-queries.log.64 does not exist
renaming /mysql/logs/slow-queries.log.63 to /mysql/logs/slow-queries.log.64 (rotatecount 120, logstart 1, i 63),
old log /mysql/logs/slow-queries.log.63 does not exist
renaming /mysql/logs/slow-queries.log.62 to /mysql/logs/slow-queries.log.63 (rotatecount 120, logstart 1, i 62),
old log /mysql/logs/slow-queries.log.62 does not exist
renaming /mysql/logs/slow-queries.log.61 to /mysql/logs/slow-queries.log.62 (rotatecount 120, logstart 1, i 61),
old log /mysql/logs/slow-queries.log.61 does not exist
renaming /mysql/logs/slow-queries.log.60 to /mysql/logs/slow-queries.log.61 (rotatecount 120, logstart 1, i 60),
old log /mysql/logs/slow-queries.log.60 does not exist
renaming /mysql/logs/slow-queries.log.59 to /mysql/logs/slow-queries.log.60 (rotatecount 120, logstart 1, i 59),
old log /mysql/logs/slow-queries.log.59 does not exist
renaming /mysql/logs/slow-queries.log.58 to /mysql/logs/slow-queries.log.59 (rotatecount 120, logstart 1, i 58),
old log /mysql/logs/slow-queries.log.58 does not exist
renaming /mysql/logs/slow-queries.log.57 to /mysql/logs/slow-queries.log.58 (rotatecount 120, logstart 1, i 57),
old log /mysql/logs/slow-queries.log.57 does not exist
renaming /mysql/logs/slow-queries.log.56 to /mysql/logs/slow-queries.log.57 (rotatecount 120, logstart 1, i 56),
old log /mysql/logs/slow-queries.log.56 does not exist
renaming /mysql/logs/slow-queries.log.55 to /mysql/logs/slow-queries.log.56 (rotatecount 120, logstart 1, i 55),
old log /mysql/logs/slow-queries.log.55 does not exist
renaming /mysql/logs/slow-queries.log.54 to /mysql/logs/slow-queries.log.55 (rotatecount 120, logstart 1, i 54),
old log /mysql/logs/slow-queries.log.54 does not exist
renaming /mysql/logs/slow-queries.log.53 to /mysql/logs/slow-queries.log.54 (rotatecount 120, logstart 1, i 53),
old log /mysql/logs/slow-queries.log.53 does not exist
renaming /mysql/logs/slow-queries.log.52 to /mysql/logs/slow-queries.log.53 (rotatecount 120, logstart 1, i 52),
old log /mysql/logs/slow-queries.log.52 does not exist
renaming /mysql/logs/slow-queries.log.51 to /mysql/logs/slow-queries.log.52 (rotatecount 120, logstart 1, i 51),
old log /mysql/logs/slow-queries.log.51 does not exist
renaming /mysql/logs/slow-queries.log.50 to /mysql/logs/slow-queries.log.51 (rotatecount 120, logstart 1, i 50),
old log /mysql/logs/slow-queries.log.50 does not exist
renaming /mysql/logs/slow-queries.log.49 to /mysql/logs/slow-queries.log.50 (rotatecount 120, logstart 1, i 49),
old log /mysql/logs/slow-queries.log.49 does not exist
renaming /mysql/logs/slow-queries.log.48 to /mysql/logs/slow-queries.log.49 (rotatecount 120, logstart 1, i 48),
old log /mysql/logs/slow-queries.log.48 does not exist
renaming /mysql/logs/slow-queries.log.47 to /mysql/logs/slow-queries.log.48 (rotatecount 120, logstart 1, i 47),
old log /mysql/logs/slow-queries.log.47 does not exist
renaming /mysql/logs/slow-queries.log.46 to /mysql/logs/slow-queries.log.47 (rotatecount 120, logstart 1, i 46),
old log /mysql/logs/slow-queries.log.46 does not exist
renaming /mysql/logs/slow-queries.log.45 to /mysql/logs/slow-queries.log.46 (rotatecount 120, logstart 1, i 45),
old log /mysql/logs/slow-queries.log.45 does not exist
renaming /mysql/logs/slow-queries.log.44 to /mysql/logs/slow-queries.log.45 (rotatecount 120, logstart 1, i 44),
old log /mysql/logs/slow-queries.log.44 does not exist
renaming /mysql/logs/slow-queries.log.43 to /mysql/logs/slow-queries.log.44 (rotatecount 120, logstart 1, i 43),
old log /mysql/logs/slow-queries.log.43 does not exist
renaming /mysql/logs/slow-queries.log.42 to /mysql/logs/slow-queries.log.43 (rotatecount 120, logstart 1, i 42),
old log /mysql/logs/slow-queries.log.42 does not exist
renaming /mysql/logs/slow-queries.log.41 to /mysql/logs/slow-queries.log.42 (rotatecount 120, logstart 1, i 41),
old log /mysql/logs/slow-queries.log.41 does not exist
renaming /mysql/logs/slow-queries.log.40 to /mysql/logs/slow-queries.log.41 (rotatecount 120, logstart 1, i 40),
old log /mysql/logs/slow-queries.log.40 does not exist
renaming /mysql/logs/slow-queries.log.39 to /mysql/logs/slow-queries.log.40 (rotatecount 120, logstart 1, i 39),
old log /mysql/logs/slow-queries.log.39 does not exist
renaming /mysql/logs/slow-queries.log.38 to /mysql/logs/slow-queries.log.39 (rotatecount 120, logstart 1, i 38),
old log /mysql/logs/slow-queries.log.38 does not exist
renaming /mysql/logs/slow-queries.log.37 to /mysql/logs/slow-queries.log.38 (rotatecount 120, logstart 1, i 37),
old log /mysql/logs/slow-queries.log.37 does not exist
renaming /mysql/logs/slow-queries.log.36 to /mysql/logs/slow-queries.log.37 (rotatecount 120, logstart 1, i 36),
old log /mysql/logs/slow-queries.log.36 does not exist
renaming /mysql/logs/slow-queries.log.35 to /mysql/logs/slow-queries.log.36 (rotatecount 120, logstart 1, i 35),
old log /mysql/logs/slow-queries.log.35 does not exist
renaming /mysql/logs/slow-queries.log.34 to /mysql/logs/slow-queries.log.35 (rotatecount 120, logstart 1, i 34),
old log /mysql/logs/slow-queries.log.34 does not exist
renaming /mysql/logs/slow-queries.log.33 to /mysql/logs/slow-queries.log.34 (rotatecount 120, logstart 1, i 33),
old log /mysql/logs/slow-queries.log.33 does not exist
renaming /mysql/logs/slow-queries.log.32 to /mysql/logs/slow-queries.log.33 (rotatecount 120, logstart 1, i 32),
old log /mysql/logs/slow-queries.log.32 does not exist
renaming /mysql/logs/slow-queries.log.31 to /mysql/logs/slow-queries.log.32 (rotatecount 120, logstart 1, i 31),
old log /mysql/logs/slow-queries.log.31 does not exist
renaming /mysql/logs/slow-queries.log.30 to /mysql/logs/slow-queries.log.31 (rotatecount 120, logstart 1, i 30),
old log /mysql/logs/slow-queries.log.30 does not exist
renaming /mysql/logs/slow-queries.log.29 to /mysql/logs/slow-queries.log.30 (rotatecount 120, logstart 1, i 29),
old log /mysql/logs/slow-queries.log.29 does not exist
renaming /mysql/logs/slow-queries.log.28 to /mysql/logs/slow-queries.log.29 (rotatecount 120, logstart 1, i 28),
old log /mysql/logs/slow-queries.log.28 does not exist
renaming /mysql/logs/slow-queries.log.27 to /mysql/logs/slow-queries.log.28 (rotatecount 120, logstart 1, i 27),
old log /mysql/logs/slow-queries.log.27 does not exist
renaming /mysql/logs/slow-queries.log.26 to /mysql/logs/slow-queries.log.27 (rotatecount 120, logstart 1, i 26),
old log /mysql/logs/slow-queries.log.26 does not exist
renaming /mysql/logs/slow-queries.log.25 to /mysql/logs/slow-queries.log.26 (rotatecount 120, logstart 1, i 25),
old log /mysql/logs/slow-queries.log.25 does not exist
renaming /mysql/logs/slow-queries.log.24 to /mysql/logs/slow-queries.log.25 (rotatecount 120, logstart 1, i 24),
old log /mysql/logs/slow-queries.log.24 does not exist
renaming /mysql/logs/slow-queries.log.23 to /mysql/logs/slow-queries.log.24 (rotatecount 120, logstart 1, i 23),
old log /mysql/logs/slow-queries.log.23 does not exist
renaming /mysql/logs/slow-queries.log.22 to /mysql/logs/slow-queries.log.23 (rotatecount 120, logstart 1, i 22),
old log /mysql/logs/slow-queries.log.22 does not exist
renaming /mysql/logs/slow-queries.log.21 to /mysql/logs/slow-queries.log.22 (rotatecount 120, logstart 1, i 21),
old log /mysql/logs/slow-queries.log.21 does not exist
renaming /mysql/logs/slow-queries.log.20 to /mysql/logs/slow-queries.log.21 (rotatecount 120, logstart 1, i 20),
old log /mysql/logs/slow-queries.log.20 does not exist
renaming /mysql/logs/slow-queries.log.19 to /mysql/logs/slow-queries.log.20 (rotatecount 120, logstart 1, i 19),
old log /mysql/logs/slow-queries.log.19 does not exist
renaming /mysql/logs/slow-queries.log.18 to /mysql/logs/slow-queries.log.19 (rotatecount 120, logstart 1, i 18),
old log /mysql/logs/slow-queries.log.18 does not exist
renaming /mysql/logs/slow-queries.log.17 to /mysql/logs/slow-queries.log.18 (rotatecount 120, logstart 1, i 17),
old log /mysql/logs/slow-queries.log.17 does not exist
renaming /mysql/logs/slow-queries.log.16 to /mysql/logs/slow-queries.log.17 (rotatecount 120, logstart 1, i 16),
old log /mysql/logs/slow-queries.log.16 does not exist
renaming /mysql/logs/slow-queries.log.15 to /mysql/logs/slow-queries.log.16 (rotatecount 120, logstart 1, i 15),
old log /mysql/logs/slow-queries.log.15 does not exist
renaming /mysql/logs/slow-queries.log.14 to /mysql/logs/slow-queries.log.15 (rotatecount 120, logstart 1, i 14),
old log /mysql/logs/slow-queries.log.14 does not exist
renaming /mysql/logs/slow-queries.log.13 to /mysql/logs/slow-queries.log.14 (rotatecount 120, logstart 1, i 13),
old log /mysql/logs/slow-queries.log.13 does not exist
renaming /mysql/logs/slow-queries.log.12 to /mysql/logs/slow-queries.log.13 (rotatecount 120, logstart 1, i 12),
old log /mysql/logs/slow-queries.log.12 does not exist
renaming /mysql/logs/slow-queries.log.11 to /mysql/logs/slow-queries.log.12 (rotatecount 120, logstart 1, i 11),
old log /mysql/logs/slow-queries.log.11 does not exist
renaming /mysql/logs/slow-queries.log.10 to /mysql/logs/slow-queries.log.11 (rotatecount 120, logstart 1, i 10),
old log /mysql/logs/slow-queries.log.10 does not exist
renaming /mysql/logs/slow-queries.log.9 to /mysql/logs/slow-queries.log.10 (rotatecount 120, logstart 1, i 9),
old log /mysql/logs/slow-queries.log.9 does not exist
renaming /mysql/logs/slow-queries.log.8 to /mysql/logs/slow-queries.log.9 (rotatecount 120, logstart 1, i 8),
old log /mysql/logs/slow-queries.log.8 does not exist
renaming /mysql/logs/slow-queries.log.7 to /mysql/logs/slow-queries.log.8 (rotatecount 120, logstart 1, i 7),
old log /mysql/logs/slow-queries.log.7 does not exist
renaming /mysql/logs/slow-queries.log.6 to /mysql/logs/slow-queries.log.7 (rotatecount 120, logstart 1, i 6),
old log /mysql/logs/slow-queries.log.6 does not exist
renaming /mysql/logs/slow-queries.log.5 to /mysql/logs/slow-queries.log.6 (rotatecount 120, logstart 1, i 5),
old log /mysql/logs/slow-queries.log.5 does not exist
renaming /mysql/logs/slow-queries.log.4 to /mysql/logs/slow-queries.log.5 (rotatecount 120, logstart 1, i 4),
old log /mysql/logs/slow-queries.log.4 does not exist
renaming /mysql/logs/slow-queries.log.3 to /mysql/logs/slow-queries.log.4 (rotatecount 120, logstart 1, i 3),
old log /mysql/logs/slow-queries.log.3 does not exist
renaming /mysql/logs/slow-queries.log.2 to /mysql/logs/slow-queries.log.3 (rotatecount 120, logstart 1, i 2),
renaming /mysql/logs/slow-queries.log.1 to /mysql/logs/slow-queries.log.2 (rotatecount 120, logstart 1, i 1),
renaming /mysql/logs/slow-queries.log.0 to /mysql/logs/slow-queries.log.1 (rotatecount 120, logstart 1, i 0),
old log /mysql/logs/slow-queries.log.0 does not exist
log /mysql/logs/slow-queries.log.121 doesn't exist -- won't try to dispose of it
renaming /mysql/logs/error.log to /mysql/logs/error.log.1
creating new log mode = 0660 uid = 27 gid = 27
renaming /mysql/logs/general.log to /mysql/logs/general.log.1
creating new log mode = 0660 uid = 27 gid = 27
renaming /mysql/logs/slow-queries.log to /mysql/logs/slow-queries.log.1
creating new log mode = 0660 uid = 27 gid = 27
running postrotate script
[root@db01 /mysql/logs]# ll
total 1675440
-rw-rw---- 1 mysql mysql          0 Aug 11  2009 error.log
-rw-rw---- 1 mysql mysql        472 Aug 11  2009 error.log.1
-rw-rw---- 1 mysql mysql        472 Aug 11 15:03 error.log.2
-rw-rw---- 1 mysql mysql        472 Aug 11 14:57 error.log.3
-rw-rw---- 1 mysql mysql          0 Aug 11  2009 error.log-old
-rw-rw---- 1 mysql mysql        213 Aug 11  2009 general.log
-rw-rw---- 1 mysql mysql        827 Aug 11  2009 general.log.1
-rw-rw---- 1 mysql mysql       1082 Aug 11 15:03 general.log.2
-rw-rw---- 1 mysql mysql      37115 Aug 11 15:01 general.log.3
-rw-rw---- 1 mysql mysql 1693930999 Aug 11 11:06 general.log.4
-rw-rw---- 1 mysql mysql        177 Aug 11  2009 slow-queries.log
-rw-rw---- 1 mysql mysql        354 Aug 11  2009 slow-queries.log.1
-rw-rw---- 1 mysql mysql        354 Aug 11 15:03 slow-queries.log.2
-rw-rw---- 1 mysql mysql   18263218 Aug 11 14:57 slow-queries.log.3
[root@db01 /mysql/logs]#

外部链接:
Rotating General Query & Slow Logs
Be careful rotating MySQL logs
Rotating logs - MySQL Administrator's Bible By Sheeri K. Cabral, Keith Murphy

5.2.6. Server Log Maintenance
5.2.3. The General Query Log

Truncating MySQL Log Files
How to logrotate mysq slow log files?
[rhelv5-list] where is mysql-log-rotate in RH EL 4/5 ?

参考MySQL某发行版本的配置文件

# This logname can be set in /etc/my.cnf
# by setting the variable "err-log"
# in the [safe_mysqld] section as follows:
#
# [safe_mysqld]
# err-log=/var/lib/mysql/mysqld.log
#
# If the root user has a password you have to create a
# /root/.my.cnf configuration file with the following
# content:
#
# [mysqladmin]
# password = <secret>
# user= root
#
# where "<secret>" is the password.
#
# ATTENTION: This /root/.my.cnf should be readable ONLY
# for root !

/var/lib/mysql/mysqld.log {
        # create 600 mysql mysql
        notifempty
        daily
        rotate 3
        missingok
        compress
    postrotate
        # just if mysqld is really running
        if test -x /usr/bin/mysqladmin && \
           /usr/bin/mysqladmin ping &>/dev/null
        then
           /usr/bin/mysqladmin flush-logs
        fi
    endscript
}

-fin-

less than stellar

2009-07-28T07:55:00.003+08:00

less than stellar

less-than-stellar
不尽如人意的,二流的

解释:
less-than-stellar
An english term meaning that something doesn't meet the expectations set in it or is less good than a similar thing coming before or after it. It is commonly used in the media and sometimes written like this: less-than-stellar.

Usage:
Those sales figures are less than stellar.
Godzilla 2000 was a less-than-stellar movie, and Dean Devlin's acting was less than stellar too.

Example from knifegirl's w/u on Falco:
Falco's third album, Falco 3, contained the single Rock Me Amadeus, which reached the top of those charts that measure such things in the U.S., Germany, Austria, and England. ... A few less-than-stellar albums followed,...

Similar terms:
substandard; dissatisfactory, half assed

例句:
In China, a Rocky Ascent for Basketball
Bonzi Wells, a less than stellar player in the N.B.A., frequently scored more than 40 points a game in the Chinese Basketball Association.

TV ratings soared, and foreign players found starring roles -- the top 15 scorers were non-Chinese, and players like Bonzi Wells and Dontae' Jones -- who had less than stellar N.B.A. careers -- frequently scored more than 40 points a game.

'Real Housewives of Atlanta' adds new face
Last season, fans of the show -- and some of the other housewives -- slammed Zolciak for her less-than-stellar voice.

-fin-