high server load, but TOP shows normal

Two instances of TOP within 5 minutes:

Code:
 PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
18432 nobody    16   0  205m  36m 7464 S 26.9  0.9  25:10.41 httpd
15008 mysql     16   0  437m 216m 3024 S 10.2  5.3  11:23.34 mysqld
13539 nobody    15   0  199m  28m 6116 S 10.2  0.7   0:00.42 httpd
14335 nobody    15   0  200m  30m 6672 S  4.9  0.7   0:00.15 httpd
14338 nobody    15   0  199m  29m 6468 S  4.3  0.7   0:00.13 httpd
14329 root      16   0 12256 4160 2856 S  3.9  0.1   0:00.12 exim
14340 nobody    15   0  199m  29m 6480 S  3.9  0.7   0:00.12 httpd
14362 root      17   0 12740 4160 2856 S  3.6  0.1   0:00.11 exim
14350 root      18   0 11740 4160 2856 S  3.3  0.1   0:00.10 exim
14337 nobody    15   0  199m  30m 6940 S  3.0  0.7   0:00.09 httpd
14341 root      20   0 11508 4160 2856 S  3.0  0.1   0:00.09 exim
14356 root      15   0 11740 4160 2852 S  3.0  0.1   0:00.09 exim
29874 nobody    15   0  201m  42m  16m S  2.6  1.0   0:10.88 httpd
14333 nobody    15   0  198m  26m 4792 S  2.6  0.7   0:00.08 httpd
13180 mysql     16   0  437m 216m 3024 S  2.0  5.3   4:43.39 mysqld
14112 nobody    15   0  199m  27m 5332 S  2.0  0.7   0:00.07 httpd
14317 root      16   0  3656 1448  772 R  2.0  0.0   0:00.13 top
14367 root      19   0 12332 4036 2760 R  2.0  0.1   0:00.06 exim
 2736 mysql     16   0  437m 216m 3024 S  1.3  5.3   1:39.47 mysqld
11094 nobody    15   0  205m  56m  27m S  1.3  1.4   1:04.02 httpd
14961 mysql     15   0  437m 216m 3024 S  1.0  5.3   2:41.27 mysqld
14991 mysql     16   0  437m 216m 3024 S  1.0  5.3   2:32.65 mysqld
13705 nobody    15   0  199m  31m 7972 S  1.0  0.8   0:00.24 httpd
14109 nobody    16   0  198m  27m 5720 S  1.0  0.7   0:00.05 httpd
11802 root      16   0  198m  26m 4676 S  0.7  0.7  11:13.96 httpd
27242 mysql     16   0  437m 216m 3024 S  0.7  5.3   4:45.59 mysqld
29693 nobody    15   0  210m  60m  27m S  0.7  1.5   1:34.36 httpd
  216 root      15   0     0    0    0 S  0.3  0.0  38:05.96 kjournald
28590 root      15   0 10084 8700 4380 S  0.3  0.2   9:24.06 cppop
28604 mailnull  16   0  7568 4512 1640 S  0.3  0.1   1:17.63 eximstats
27316 root      15   0 11868  10m 5116 S  0.3  0.3   0:16.81 cpsrvd
14999 mysql     15   0  437m 216m 3024 S  0.3  5.3   3:56.85 mysqld
Code:
  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
18432 nobody    15   0  205m  36m 7464 R 25.7  0.9  25:46.62 httpd
15008 mysql     15   0  437m 218m 3024 R  9.3  5.4  11:37.44 mysqld
13576 nobody    15   0  200m  33m 9632 S  7.4  0.8   0:01.52 httpd
15291 nobody    15   0  199m  31m 8036 S  7.1  0.8   0:00.56 httpd
15911 nobody    19   0  200m  30m 7076 S  7.1  0.8   0:00.22 httpd
29651 nobody    15   0  212m  64m  27m S  4.5  1.6   2:11.49 httpd
15958 root      19   0 12656 4164 2856 S  4.2  0.1   0:00.13 exim
15949 root      20   0 11808 4160 2856 S  3.5  0.1   0:00.11 exim
15953 root      20   0 13244 4164 2856 S  3.5  0.1   0:00.11 exim
29652 nobody    15   0  211m  62m  27m S  3.2  1.5   2:08.08 httpd
15937 root      17   0 11656 4132 2852 S  3.2  0.1   0:00.10 exim
15951 mailnull  15   0  8764 3568 2528 S  3.2  0.1   0:00.10 exim
29880 nobody    15   0  209m  54m  21m S  2.9  1.3   0:14.16 httpd
 5972 nobody    15   0  203m  45m  17m S  2.9  1.1   0:18.30 httpd
10791 nobody    15   0  201m  41m  16m S  2.9  1.0   0:04.23 httpd
15963 root      18   0 13048 4280 2904 S  2.9  0.1   0:00.09 exim
29693 nobody    15   0  210m  60m  27m S  2.6  1.5   1:36.17 httpd
15923 mailnull  15   0  9032 3868 2592 S  2.6  0.1   0:00.08 exim
13200 mysql     16   0  437m 218m 3024 S  1.6  5.4   3:37.26 mysqld
14999 mysql     15   0  437m 218m 3024 S  1.6  5.4   4:23.42 mysqld
15793 root      17   0  4164 1456  776 R  1.6  0.0   0:00.35 top
15907 nobody    16   0  198m  29m 6824 S  1.3  0.7   0:00.04 httpd
15913 nobody    15   0  199m  32m 8788 S  1.3  0.8   0:00.13 httpd
29662 nobody    16   0  207m  60m  29m S  1.0  1.5   1:37.04 httpd
29692 nobody    15   0  211m  62m  28m S  1.0  1.5   1:42.26 httpd
29654 nobody    15   0  211m  63m  28m S  0.6  1.6   1:29.33 httpd
    2 root      RT   0     0    0    0 S  0.3  0.0   0:07.36 migration/0

I can't seem to trace why the load is floating around 7 / 8 CPU average.

Specs are:

Dual Xeon 2.8GHz
4GB Memory
100MBit Port
300GB SCSI drives (21% full)
5GB /tmp partition (15% full)

PHP 4.4.2 w/ zend optimizer and eaccelerator
MySQL 4.1
Apache 1.3x

About 128 IP's on this server (resellers)

/etc/my.cnf

Code:
#DO NOT MODIFY THE FOLLOWING COMMENTED LINES!
#Created with ELS from www.nsonetworks.com
#els-build=4.1
[mysqld]
datadir=/var/lib/mysql
skip-locking
#skip-networking
safe-show-database
query_cache_limit=1M
query_cache_size=128M ## 32MB for every 1GB of RAM
query_cache_type=1
max_user_connections=200
max_connections=500
interactive_timeout=10
wait_timeout=20
connect_timeout=20
thread_cache_size=128
key_buffer=256M ## 64MB for every 1GB of RAM
join_buffer=1M
max_connect_errors=20
max_allowed_packet=16M
table_cache=1024
record_buffer=1M
sort_buffer_size=4M ## 1MB for every 1GB of RAM
read_buffer_size=4M ## 1MB for every 1GB of RAM
read_rnd_buffer_size=4M  ## 1MB for every 1GB of RAM
thread_concurrency=8 ## Number of CPUs x 2
myisam_sort_buffer_size=64M
server-id=1
log_slow_queries=/var/log/mysql-slow-queries.log
long_query_time=2
collation-server=latin1_general_ci
old-passwords

[mysql.server]
user=mysql
basedir=/var/lib

[safe_mysqld]
err-log=/var/log/mysqld.log
pid-file=/var/lib/mysql/mysql.pid
open_files_limit=8192

[mysqldump]
quick
max_allowed_packet=16M

[mysql]
no-auto-rehash
#safe-updates

[isamchk]
key_buffer=256M
sort_buffer=64M
read_buffer=16M
write_buffer=16M

[myisamchk]
key_buffer=256M
sort_buffer=64M
read_buffer=16M
write_buffer=16M

[mysqlhotcopy]
interactive-timeout
httpd.conf file

Timeout 300
KeepAlive On
MaxKeepAliveRequests 100
KeepAliveTimeout 15
MinSpareServers 5
MaxSpareServers 10
StartServers 5
MaxRequestsPerChild 0

php.ini file (resource limits)

max_execution_time = 30
memory_limit = 8M
post_max_size = 55M

 

 

 

 

Top