Awk craziness: Processing log files

In: General|Linux

13 Apr 2013

A while back I wrote an awk script to process a custom log file to work out how an application was performing over a period of time.
The following script takes a timestamp in field one $1 and also an arbitrary field $10 in this case.

For every 100 lines of the log file processed it would calculate the min, max, avg, stddev for rolling stats.

zcat Summary.2012-12-09.gz | grep '2012-12-09 20:[234]' | awk '
  BEGIN{
    size=50;
    tmax = -999999999;
    tmin = 999999999;
  }
  {   # Accumulate basic data
      cnt[$10]++;
      item[++n] = $10;
      if ($10 > max) max = $10; if ($10 < min) min = $10;
      if ($10 > tmax) tmax = $10; if ($10 < tmin) tmin = $10;
  }
  {
    mod=NR%size;
    if(NR<=size) {
      count++;
    } else {
      sum-=array[mod];
      min=max=$10;
      
    };
    total+=$10;
    sum+=$10; sumsq+=$10*$10;
    array[mod]=$10;  
    print $1" "$2 "\tvalue: " $10 "\tmin: " min "\tmax: " max "\tavg: " sum/count "\tstddev: " sqrt(sumsq/NR - (sum/NR)**2)  "\t"
  }
  END {
    printf("\n\n# Lines processed = %d\n", NR);
    printf("# Total = %d\n", total);
    printf("# Sqrt = %d\n", sqrt(sumsq/NR - (sum/NR)**2));
    # Print Descriptive Statistics
    printf("# Count = %d\n", n);
    printf("# Min = %d\n", tmin);
    decile = 1;
    for (decile = 10; decile < 100; decile += 10) {
      idx = int((decile * n) / 100) + 1;
      printf("# %d%% decile = %d\n", decile, item[idx]);
      if (decile == 50)
        median = item[idx];
    }
    printf("# Max = %d\n", tmax);
 
    printf("# Median = %d\n", median);
    for (i in cnt) {
      if (cnt[i] > cnt[mode])
        mode = i;
    }
    printf("# Mode = %d\n", mode);
  }
' | less
 
...
2012-12-09 20:20:01     value: 18       min:    max: 18 avg: 5.8        stddev: 6.20967 
2012-12-09 20:20:01     value: 7        min:    max: 18 avg: 6  stddev: 5.68624 
2012-12-09 20:20:01     value: 7        min:    max: 18 avg: 6.14286    stddev: 5.27605 
2012-12-09 20:20:01     value: 4        min:    max: 18 avg: 5.875      stddev: 4.98592 
2012-12-09 20:20:01     value: 2        min:    max: 18 avg: 5.44444    stddev: 4.85595 
2012-12-09 20:20:02     value: 3        min:    max: 18 avg: 5.2        stddev: 4.66476 
2012-12-09 20:20:02     value: 4        min:    max: 18 avg: 5.09091    stddev: 4.46103 
2012-12-09 20:20:02     value: 104      min:    max: 104        avg: 13.3333    stddev: 27.6687 
2012-12-09 20:20:02     value: 2        min:    max: 104        avg: 12.4615    stddev: 26.7542 
2012-12-09 20:20:02     value: 25       min:    max: 104        avg: 13.3571    stddev: 25.9824 
2012-12-09 20:20:03     value: 3        min:    max: 104        avg: 12.6667    stddev: 25.234  
2012-12-09 20:20:03     value: 3        min:    max: 104        avg: 12.0625    stddev: 24.5445 
2012-12-09 20:20:03     value: 3        min:    max: 104        avg: 11.5294    stddev: 23.907  
2012-12-09 20:20:03     value: 2        min:    max: 104        avg: 11 stddev: 23.3357 
2012-12-09 20:20:03     value: 6        min:    max: 104        avg: 10.7368    stddev: 22.7407 
2012-12-09 20:20:04     value: 4        min:    max: 104        avg: 10.4       stddev: 22.2135 
2012-12-09 20:20:04     value: 3        min:    max: 104        avg: 10.0476    stddev: 21.7354 
2012-12-09 20:20:04     value: 2        min:    max: 104        avg: 9.68182    stddev: 21.3017 
2012-12-09 20:20:04     value: 70       min:    max: 104        avg: 12.3043    stddev: 24.1938 
2012-12-09 20:20:04     value: 4        min:    max: 104        avg: 11.9583    stddev: 23.7425 
2012-12-09 20:20:04     value: 12       min:    max: 104        avg: 11.96      stddev: 23.2628 
2012-12-09 20:20:04     value: 14       min:    max: 104        avg: 12.0385    stddev: 22.8144 
2012-12-09 20:20:04     value: 6        min:    max: 104        avg: 11.8148    stddev: 22.417  
2012-12-09 20:20:04     value: 5        min:    max: 104        avg: 11.5714    stddev: 22.0493 
2012-12-09 20:20:04     value: 2        min:    max: 104        avg: 11.2414    stddev: 21.7361 
2012-12-09 20:20:04     value: 30       min:    max: 104        avg: 11.8667    stddev: 21.6344 
2012-12-09 20:20:05     value: 5        min:    max: 104        avg: 11.6452    stddev: 21.3172 
2012-12-09 20:20:05     value: 22       min:    max: 104        avg: 11.9688    stddev: 21.0587 
2012-12-09 20:20:05     value: 4        min:    max: 104        avg: 11.7273    stddev: 20.7821 
2012-12-09 20:20:05     value: 2        min:    max: 104        avg: 11.4412    stddev: 20.54   
2012-12-09 20:20:05     value: 25       min:    max: 104        avg: 11.8286    stddev: 20.3701 
2012-12-09 20:20:05     value: 27       min:    max: 104        avg: 12.25      stddev: 20.2394 
2012-12-09 20:20:05     value: 3        min:    max: 104        avg: 12 stddev: 20.0203 
2012-12-09 20:20:06     value: 2        min:    max: 104        avg: 11.7368    stddev: 19.8198 
2012-12-09 20:20:06     value: 1        min:    max: 104        avg: 11.4615    stddev: 19.6375 
2012-12-09 20:20:06     value: 2        min:    max: 104        avg: 11.225     stddev: 19.4467 
2012-12-09 20:20:06     value: 36       min:    max: 104        avg: 11.8293    stddev: 19.5846 
2012-12-09 20:20:06     value: 3        min:    max: 104        avg: 11.619     stddev: 19.3968 
2012-12-09 20:20:06     value: 27       min:    max: 104        avg: 11.9767    stddev: 19.3096 
2012-12-09 20:20:06     value: 2        min:    max: 104        avg: 11.75      stddev: 19.1467 
2012-12-09 20:20:07     value: 5        min:    max: 104        avg: 11.6       stddev: 18.9589 
2012-12-09 20:20:07     value: 2        min:    max: 104        avg: 11.3913    stddev: 18.8039 
2012-12-09 20:20:07     value: 5        min:    max: 104        avg: 11.2553    stddev: 18.6256 
2012-12-09 20:20:07     value: 4        min:    max: 104        avg: 11.1042    stddev: 18.4597 
2012-12-09 20:20:07     value: 2        min:    max: 104        avg: 10.9184    stddev: 18.3156 
2012-12-09 20:20:07     value: 4        min:    max: 104        avg: 10.78      stddev: 18.1574 
...
 
# Lines processed = 9863
# Total = 80137
# Sqrt = 15
# Count = 9863
# Min = 1
# 10% decile = 8
# 20% decile = 1
# 30% decile = 3
# 40% decile = 3
# 50% decile = 17
# 60% decile = 7
# 70% decile = 5
# 80% decile = 1
# 90% decile = 4
# Max = 140
# Median = 17
# Mode = 2

Share

Elastic Search Presentation

In: General|PHP

29 Oct 2012

I’ve implemented a number of sites with Elastic Search and recently had a two-day spike to implement Elastic Search for Photobox’s backoffice systems. As a result I have since wrote a presentation and presented at Photobox on Friday. I have posted the slides for the presentation here…

I wrote this presentation using rvl.io – reveal.js, however unfortunately does not allow you to nicely export the presentation to pdf or other non html formats. Best viewed full screen…

A few other interesting presentations I came across whilst writing up these slides.

Share

Requests per second from Apache logs

In: General|Linux

22 Sep 2012

Whilst recently looking into usage of SSL, I needed to analyse some logs from apache to determine requests per second.
Initially I wrote a quick bash script to do this, however figured it would be handy to elaborate on this, as a result I have posted a perl script on github at apache-log-stats

    total_requests=$(zcat b2f01_ApacheAccess.2011-12-05.gz | \
    sort -b -k4.9,4.12 -k4.5b,4.7Mb -k4.2,4.3 -k4.14,4 | \
    awk -F '[ [/:]' '{print $7"-"$6"-"$5":"$8":"$9":"$10}' | \
    sort | uniq -c  | sort -g | tail -n1);

    echo $total_requests;

And the perl script.

andrew@andrew-OptiPlex-7010:/var/www/logs/bin$ ./apache-log-stat $(ls ../logs/b2f*_ApacheAccess.2011-12-05.gz)
Processing
====================

../logs/b2f01_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.92    variance_requests_per_second: 25.93 total_requests:  413603 stddev_requests_per_second: 5.09    max_requests_per_second:   57   
../logs/b2f02_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 6.05    variance_requests_per_second: 26.55 total_requests:  417711 stddev_requests_per_second: 5.15    max_requests_per_second:   45   
../logs/b2f03_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 6.09    variance_requests_per_second: 26.64 total_requests:  419073 stddev_requests_per_second: 5.16    max_requests_per_second:   48   
../logs/b2f04_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.07    variance_requests_per_second: 12.82 total_requests:  325374 stddev_requests_per_second: 3.58    max_requests_per_second:   29   
../logs/b2f05_ApacheAccess.2011-12-05.gz - avg_requests_per_second:    0    variance_requests_per_second:    0  total_requests:    0    stddev_requests_per_second:    0    max_requests_per_second:    0   
../logs/b2f06_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.20    variance_requests_per_second: 13.29 total_requests:  338946 stddev_requests_per_second: 3.65    max_requests_per_second:   30   
../logs/b2f07_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.09    variance_requests_per_second: 12.92 total_requests:  330180 stddev_requests_per_second: 3.59    max_requests_per_second:   28   
../logs/b2f08_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.08    variance_requests_per_second: 12.78 total_requests:  334373 stddev_requests_per_second: 3.57    max_requests_per_second:   33   
../logs/b2f09_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.05    variance_requests_per_second: 12.66 total_requests:  327368 stddev_requests_per_second: 3.56    max_requests_per_second:   28   
../logs/b2f10_ApacheAccess.2011-12-05.gz - avg_requests_per_second:    0    variance_requests_per_second:    0  total_requests:    0    stddev_requests_per_second:    0    max_requests_per_second:    0   
../logs/b2f11_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.51    variance_requests_per_second: 15.00 total_requests:  370115 stddev_requests_per_second: 3.87    max_requests_per_second:   35   
../logs/b2f12_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.43    variance_requests_per_second: 14.41 total_requests:  357580 stddev_requests_per_second: 3.80    max_requests_per_second:   37   
../logs/b2f13_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.47    variance_requests_per_second: 15.09 total_requests:  366275 stddev_requests_per_second: 3.88    max_requests_per_second:   36   
../logs/b2f14_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.39    variance_requests_per_second: 14.51 total_requests:  363630 stddev_requests_per_second: 3.81    max_requests_per_second:   34   
../logs/b2f15_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.40    variance_requests_per_second: 14.40 total_requests:  361463 stddev_requests_per_second: 3.80    max_requests_per_second:   41   
../logs/b2f16_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.87    variance_requests_per_second: 17.03 total_requests:  403126 stddev_requests_per_second: 4.13    max_requests_per_second:   48   
../logs/b2f17_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.90    variance_requests_per_second: 16.81 total_requests:  404777 stddev_requests_per_second: 4.10    max_requests_per_second:   37   
../logs/b2f18_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.89    variance_requests_per_second: 16.89 total_requests:  407635 stddev_requests_per_second: 4.11    max_requests_per_second:   30   
../logs/b2f19_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.82    variance_requests_per_second: 16.13 total_requests:  402808 stddev_requests_per_second: 4.02    max_requests_per_second:   36   
../logs/b2f20_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 5.84    variance_requests_per_second: 16.44 total_requests:  404247 stddev_requests_per_second: 4.05    max_requests_per_second:   35   
../logs/b2f21_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 7.91    variance_requests_per_second: 29.21 total_requests:  583211 stddev_requests_per_second: 5.40    max_requests_per_second:   41   
../logs/b2f22_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 7.86    variance_requests_per_second: 28.31 total_requests:  575122 stddev_requests_per_second: 5.32    max_requests_per_second:   38   
../logs/b2f23_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 7.68    variance_requests_per_second: 27.37 total_requests:  571349 stddev_requests_per_second: 5.23    max_requests_per_second:   39   
../logs/b2f24_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 7.76    variance_requests_per_second: 27.80 total_requests:  571728 stddev_requests_per_second: 5.27    max_requests_per_second:   37   
../logs/b2f25_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 7.90    variance_requests_per_second: 28.59 total_requests:  583242 stddev_requests_per_second: 5.35    max_requests_per_second:   51   
../logs/b2f26_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 7.87    variance_requests_per_second: 28.02 total_requests:  576603 stddev_requests_per_second: 5.29    max_requests_per_second:   36   
../logs/b2f27_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 7.91    variance_requests_per_second: 29.45 total_requests:  584972 stddev_requests_per_second: 5.43    max_requests_per_second:   42   
../logs/b2f28_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 7.66    variance_requests_per_second: 27.15 total_requests:  561073 stddev_requests_per_second: 5.21    max_requests_per_second:   38   
../logs/b2f29_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 7.83    variance_requests_per_second: 28.15 total_requests:  578100 stddev_requests_per_second: 5.31    max_requests_per_second:   47   
../logs/b2f30_ApacheAccess.2011-12-05.gz - avg_requests_per_second: 7.74    variance_requests_per_second: 27.62 total_requests:  574167 stddev_requests_per_second: 5.26    max_requests_per_second:   39   
Total:  avg_requests_per_second: 178.19 total_requests: 12507851    variance_requests_per_second: 581.99    stddev_requests_per_second: 125.99  max_requests_per_second: 1075   


Requests per hour
====================

   00 #                                                                                                           199683 / 12507851 (  1.60% )  var: 12308168   rqs:  55    rqm: 3328
   01                                                                                                             121028 / 12507851 (  0.97% )  var: 12386823   rqs:  33    rqm: 2017
   02                                                                                                              92839 / 12507851 (  0.74% )  var: 12415012   rqs:  25    rqm: 1547
   03                                                                                                              79303 / 12507851 (  0.63% )  var: 12428548   rqs:  22    rqm: 1321
   04                                                                                                              76682 / 12507851 (  0.61% )  var: 12431169   rqs:  21    rqm: 1278
   05                                                                                                              96342 / 12507851 (  0.77% )  var: 12411509   rqs:  26    rqm: 1605
   06 #                                                                                                           141415 / 12507851 (  1.13% )  var: 12366436   rqs:  39    rqm: 2356
   07 #                                                                                                           245247 / 12507851 (  1.96% )  var: 12262604   rqs:  68    rqm: 4087
   08 ###                                                                                                         431089 / 12507851 (  3.45% )  var: 12076762   rqs: 119    rqm: 7184
   09 ####                                                                                                        589031 / 12507851 (  4.71% )  var: 11918820   rqs: 163    rqm: 9817
   10 #####                                                                                                       662950 / 12507851 (  5.30% )  var: 11844901   rqs: 184    rqm: 11049
   11 #####                                                                                                       637200 / 12507851 (  5.09% )  var: 11870651   rqs: 177    rqm: 10620
   12 #####                                                                                                       739012 / 12507851 (  5.91% )  var: 11768839   rqs: 205    rqm: 12316
   13 ######                                                                                                      780787 / 12507851 (  6.24% )  var: 11727064   rqs: 216    rqm: 13013
   14 ######                                                                                                      779051 / 12507851 (  6.23% )  var: 11728800   rqs: 216    rqm: 12984
   15 #####                                                                                                       667872 / 12507851 (  5.34% )  var: 11839979   rqs: 185    rqm: 11131
   16 ######                                                                                                      750655 / 12507851 (  6.00% )  var: 11757196   rqs: 208    rqm: 12510
   17 ######                                                                                                      804304 / 12507851 (  6.43% )  var: 11703547   rqs: 223    rqm: 13405
   18 #####                                                                                                       740101 / 12507851 (  5.92% )  var: 11767750   rqs: 205    rqm: 12335
   19 ######                                                                                                      795684 / 12507851 (  6.36% )  var: 11712167   rqs: 221    rqm: 13261
   20 #######                                                                                                     973406 / 12507851 (  7.78% )  var: 11534445   rqs: 270    rqm: 16223
   21 #######                                                                                                     979602 / 12507851 (  7.83% )  var: 11528249   rqs: 272    rqm: 16326
   22 #####                                                                                                       724108 / 12507851 (  5.79% )  var: 11783743   rqs: 201    rqm: 12068
   23 ###                                                                                                         400460 / 12507851 (  3.20% )  var: 12107391   rqs: 111    rqm: 6674


Comparison by distribution over an hour
====================

   00 ####################                                                                                      199683 / 979602 ( 20.38% )  var: 779919 rqs:  55    rqm: 3328
   01 ############                                                                                              121028 / 979602 ( 12.35% )  var: 858574 rqs:  33    rqm: 2017
   02 #########                                                                                                  92839 / 979602 (  9.48% )  var: 886763 rqs:  25    rqm: 1547
   03 ########                                                                                                   79303 / 979602 (  8.10% )  var: 900299 rqs:  22    rqm: 1321
   04 #######                                                                                                    76682 / 979602 (  7.83% )  var: 902920 rqs:  21    rqm: 1278
   05 #########                                                                                                  96342 / 979602 (  9.83% )  var: 883260 rqs:  26    rqm: 1605
   06 ##############                                                                                            141415 / 979602 ( 14.44% )  var: 838187 rqs:  39    rqm: 2356
   07 #########################                                                                                 245247 / 979602 ( 25.04% )  var: 734355 rqs:  68    rqm: 4087
   08 ############################################                                                              431089 / 979602 ( 44.01% )  var: 548513 rqs: 119    rqm: 7184
   09 ############################################################                                              589031 / 979602 ( 60.13% )  var: 390571 rqs: 163    rqm: 9817
   10 ###################################################################                                       662950 / 979602 ( 67.68% )  var: 316652 rqs: 184    rqm: 11049
   11 #################################################################                                         637200 / 979602 ( 65.05% )  var: 342402 rqs: 177    rqm: 10620
   12 ###########################################################################                               739012 / 979602 ( 75.44% )  var: 240590 rqs: 205    rqm: 12316
   13 ###############################################################################                           780787 / 979602 ( 79.70% )  var: 198815 rqs: 216    rqm: 13013
   14 ###############################################################################                           779051 / 979602 ( 79.53% )  var: 200551 rqs: 216    rqm: 12984
   15 ####################################################################                                      667872 / 979602 ( 68.18% )  var: 311730 rqs: 185    rqm: 11131
   16 ############################################################################                              750655 / 979602 ( 76.63% )  var: 228947 rqs: 208    rqm: 12510
   17 ##################################################################################                        804304 / 979602 ( 82.11% )  var: 175298 rqs: 223    rqm: 13405
   18 ###########################################################################                               740101 / 979602 ( 75.55% )  var: 239501 rqs: 205    rqm: 12335
   19 #################################################################################                         795684 / 979602 ( 81.23% )  var: 183918 rqs: 221    rqm: 13261
   20 ###################################################################################################       973406 / 979602 ( 99.37% )  var:   6196 rqs: 270    rqm: 16223
   21 ####################################################################################################      979602 / 979602 (100.00% )  var:      0 rqs: 272    rqm: 16326
   22 #########################################################################                                 724108 / 979602 ( 73.92% )  var: 255494 rqs: 201    rqm: 12068
   23 ########################################

Share

Hackday @ Everlution

In: General|Linux|MySQL|PHP

25 Jul 2012

At Everlution we did a hack day to help motivate developers and to get the team working together. We started at 8.30am and finished at 5pm. It was pretty impressive what all the teams managed to achieve, with four teams of two developers and a designer.

It was set out in a competition style event with all teams competing against each other with a defined set of points for each task achieved. The tasks were very ambitious considering we had a day to complete the task.

The key task was to analyse a webcam and determine whether a meeting room was in use or not by analysing the web cams jpeg stream.
The motivation for which is that many staff members book meeting rooms and don’t use them or randomly hijack meeting rooms.

Task sheet/Points

# Task Points*
1 To build a webpage that tells the user if the boardroom is in use or not by parsing the JPEG stream emitted from the webcam.
Scoring:

  • Web page displays in real time whether Boardroom is in use or not.
  • Using own parsing algorithm for detection and not an off-the-shelf library
  • Detection system to not trigger with someone going in/out of Studio 2
  • +35 points
  • +15 points
  • +10 points
2 To use a Face detection and recognition library to say whether one member of your team is in the Boardroom or not.
  • +20 points
3 To build an MSN or Skype Bot that will answer a natural language question on whether the Boardroom is in use or not. Must be able to handle at least 3 variations of how question could be asked.
Scoring:

  • MSN bot or Skype bot:
  • Using own parsing algorithm for detection and not an off-the-shelf library
  • Also answers whether one of your team is in the room or not
  • +15 points
  • +10 points
4 To build a native mobile app that will display same information.
Scoring:

  • Android or iPhone
  • +10 points
5 To come up with a name, brand/logo and design for the service.
Scoring:

  • Design
  • Build/Implementation
  • +15 points
  • +10 points
6 Bonus: Good team work
  • +5 points
7 Bonus: Quality, reliability and elegance of solution
  • +5 points
8 Sabotaging another team’s effort, being disruptive, not participating in spirit of competition, etc.
  • -150 points

With regards to “Detection system to not trigger with someone going in/out of Studio 2″. There is a door going into the boardroom and a door just outside going into another office of Everlution. So any body outside the glass should not be detected going into that office.

The implementations varied drastically between teams, some based on motion.

Team 1 – My Team:

Used a C# client application to detect motion in real time from the remote image and/or directly attached camera using AForge.

The sample client was more than enough and simply ran a timed event to push the stats across over http to a PHP based API and store within a MySQL database. The web page then polled periodically to check the availability from the MySQL database, which produces the following data set. If the motion_level was above 0.02 it would indicate the room is occupied. This histogram is indicated at the bottom of the image stream.

The main issue was that it took me till 11am to install Visual Studio. Additionally the implementation was improved by adjusting the motion detection method depending on whether the stream was sourced from the camera itself or by the image, which was updated every two seconds.

{
   "boardroom" : {
      "data" : {
         "date_created" : "2012-07-24 00:14:19",
         "name" : "boardroom",
         "signal_time" : "24/07/2012 00:14:20",
         "fqdn" : "HP28458242993",
         "motion_level" : "0.000406901",
         "detected_objects_count" : "-1",
         "has_motion_been_detected" : "",
         "id" : "19945",
         "motion_alarm_level" : "0.015",
         "flash" : "0"
      },
      "last_used" : {
         "date_created" : "2012-07-24 00:14:19",
         "name" : "boardroom",
         "signal_time" : "24/07/2012 00:14:20",
         "fqdn" : "HP28458242993",
         "motion_level" : "0.000406901",
         "detected_objects_count" : "-1",
         "has_motion_been_detected" : "",
         "id" : "19945",
         "motion_alarm_level" : "0.015",
         "flash" : "0"
      },
      "is_free" : true
   }
}

AForge is an excellent library and provides the following…

  • AForge.Imaging – library with image processing routines and filters
  • AForge.Vision – computer vision library
  • AForge.Neuro – neural networks computation library
  • AForge.Genetic – evolution programming library
  • AForge.Fuzzy – fuzzy computations library
  • AForge.MachineLearning – machine learning library
  • AForge.Robotics – library providing support of some robotics kits
  • AForge.Video – set of libraries for video processing

In order to achieve face detection, I was going to use OpenCV, however ran out of time to fully implement. As such we implemented face.com to attempt face recognition. Unfortunately for all implementations this API failed to recognise anyone.

Additionally an MSN bot was built on put on top although did not implement any real natural language implementation.

The results in action



Team 2:

The implementation was based on PHP Imagick using edge detection by enhancing the images with edgeImage and creating composite images.

Two sets of metrics were extracted for statistical analysis from the image. For all channels the mean, stddev from each image used (The raw image, the composite). A histogram was also generated for each channel with the std dev, variance and skew.

Team 2 were to use two important metrics that they did not manage to implement, however their intention was too…

These stats were then passed to Quinlan’s C4.5 and provided with some training pictures with people in the room or not.

The idea behind applying edge density & energy analysis was for the algorithm to learn about the room’s layout… i.e. moving a chair would not change density nor energy a lot, but adding another one would do, so theoretically this approach should learn about objects inside the room. Filtering by edge density to detect features would remove unimportant (not dense) objects such as chairs or people being too far away.

If C4.5 used that algorithm with enough data with people and other objects, it would learn itself about how to distinguish between a face and a chair.

Team 3:

Wrote their implementation using python with bindings for OpenCV. Initially loading the image creating 3 grayscale images.

  • Image 1: Last board room image
  • Image 2: The current board room image
  • Image 3: The AbsDiff between Image 1 and Image 2

Using ImageChops.difference the stats for the difference are stored in sqlite and divided by an average. If the condition is met it will mark the room as being occupied.

Snapshots from OpenCV can be seen below.

Team 4:

Used PHPs Imagick extension and the following function http://robert-lerner.com/imagecompare.php. They ran a series of slices whether someone was in the room and determined that if there was a 5% variance the meeting room was occupied.

The results in action

Other references

C Sharp / C / C++

Realtime Webcam Sudoku Solver
Motion Detection Algorithms
Face Detection in C#
Multiple face detection and recognition in real time
Emgu CV
Hands Gesture Recognition
Playing Card Recognition Using AForge.Net Framework

OpenCV

Feature detection
Motion Analysis and Object Tracking
Canny Edge Detector
Laplace Operator
Sobel Derivatives

Share

apt-get unattended / non-interactive installation

In: General|Linux

19 Jan 2012

I previously wrote about “Installing Java & MySQL – unattended/non-interactive installation”. Whilst using slack, I have been running apt-get during postinstall rather than preinstall.
Files are transferred from slack after preinstall as such transferring any files that conflict with the packages configuration files, apt-get will prompt regardless of DEBIAN_FRONTEND=noninteractive. Not ideal if your trying to run non-interactive. You can use the -o option to circumvent this…

apt-get -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold'  -f -q -y php-cli

Previous example of non-interactive installations.

INSTALLER_LOG=/var/log/non-interactive-installer.log

installnoninteractive(){
    sudo bash -c "DEBIAN_FRONTEND=noninteractive aptitude install -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold'  -f -q -y $* >> $INSTALLER_LOG"
}

assert_package() {
    is_installed=$(apt-cache policy "$1" | grep 'Installed: (none)' | wc -l);
    if [ "$is_installed" -eq "1" ] ; then 
        #apt-get install -q -y $1;
        installnoninteractive $1;
    fi; 
}

Share

PHP apache_note, Graphite & Logster for logging

In: General|Linux|PHP

9 Jan 2012

A nice tip for logging data against apache logs within requests. Apache_note is a wrapper for Apache’s table_get and table_set. It edits the table of notes that exists during a request. The table’s purpose is to allow Apache modules to communicate.

if (function_exists('apache_note')) {

    if (isset($_SESSION) && !empty($_SESSION)) {
        apache_note('session_id',     session_id());
        apache_note('users_id',       isset($_SESSION['users_id'])? $_SESSION['users_id'] : null);
    }
    apache_note('start_time',     isset($_SERVER['REQUEST_TIME'])? $_SERVER['REQUEST_TIME'] : null);
    apache_note('end_time',       time());
    apache_note('peak_memory',    memory_get_peak_usage());

}

This will allow the variable to be exposed to apache logs in the form of the following.

LogFormat "%{session_id}n %{users_id}n " combined

You can easily add this globally by using the auto_append_file directive in the php ini.

auto_append_file = /var/www/footer.php

Additionally it is quite handy to include the following in a custom log format for debugging purposes. For example if you have hung apache processes.

Format String Description
%…D The time taken to serve the request, in microseconds.
%…B Size of response in bytes, excluding HTTP headers.
%…P The process ID of the child that serviced the request.
%…{format}P The process ID or thread id of the child that serviced the request. Valid formats are pid and tid. (Apache 2.0.46 and later)
%…X Connection status when response is completed:
X = connection aborted before the response completed.
+ = connection may be kept alive after the response is sent.
- = connection will be closed after the response is sent.
(This directive was %…c in late versions of Apache 1.3, but this conflicted with the historical ssl %…{var}c syntax.)

Common Log Format with Virtual Host and above params

LogFormat "%v %h %l %u %t \"%r\" %>s %b %B %P %X %{session_id}n %{users_id}n %{peak_memory}n" custom_log

You can take this one step further and log these in realtime using Graphite, Logtailer and logster. This holds the offset from the last time you scanned your logs. So this will give you accurate graphs. I have also added a custom parser for PHP, which can be found here.

An example of this, although does not have any errors shown below (hidden the server names).

Share

Vagrant – Automating PHP/MySQL Installation with bash/slack for continuous deployment

In: General|Linux|MySQL|PHP

11 Dec 2011

Vagrant is simply a wrapper around Virtualbox headless that allows for provisioning virtual machines with support for puppet, chef-solo, chef, and bash. This allows you to automate the deployment and sandboxing of development sites. Additional base box images can be found at vagrantbox.es.

Installing Vagrant

apt-get install -y ruby1.9.1 ruby1.9.1-dev
ln -svf /usr/bin/ruby1.9.1 /etc/alternatives/ruby

gem install vagrant
vagrant box add base http://puppetlabs.s3.amazonaws.com/pub/squeeze64.box

mkdir ~/vagrant/test && ~/vagrant/test;
vagrant init base

Change the config for the virtual box instance to be bootstrapped with a bash script.

echo "
Vagrant::Config.run do |config|
  config.vm.box = "base"

  config.vm.define :web do |web_config|
    web_config.vm.box = "base"
    web_config.vm.forward_port("http", 80, 28080)
    web_config.vm.provision :shell, :path => "slack.sh web"
  end
end
" > Vagrantfile

On a remote/local server configure slack, hit enter on prompts to use defaults for ssh-keygen and use without a passphrase.

useradd -m slack
ssh-keygen -f~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub

Replace the contents of the RSA file in the following slack.sh file in place of <place_rsa_key_contents_here>. Additionally change the ~/.ssh/config ip in Hostname to the relevant ip to where slack is enabled.


#!/bin/bash
echo "export DEBIAN_FRONTEND=noninteractive" > ~/base.install.sh
echo "apt-get -q update" >> ~/base.install.sh
#echo "apt-get -q -y upgrade" >> ~/base.install.sh
echo "apt-get -q -y install slack locales-all" >> ~/base.install.sh
echo "mkdir -p ~/.ssh;" >> ~/base.install.sh

echo "
cat > ~/.ssh/slack.id_rsa < <-EOD
-----BEGIN RSA PRIVATE KEY-----
<place_rsa_key_contents_here>
-----END RSA PRIVATE KEY-----
EOD
" >> ~/base.install.sh
echo "chmod 0600 ~/.ssh/slack.id_rsa" >> ~/base.install.sh

echo "
cat > ~/.ssh/config < <-EOD
Host slack
  HostName 192.168.0.1
  User slack
  Port 22
  UserKnownHostsFile /dev/null
  StrictHostKeyChecking no
  PasswordAuthentication no
  IdentityFile ~/.ssh/slack.id_rsa
  IdentitiesOnly yes
EOD
" >> ~/base.install.sh

echo "Installing";
sudo -i chmod +x ~/base.install.sh
sudo -i ~/base.install.sh | tee install.log

slack --verbose --source slack:~/project web

At this point you have enough to start the bootstrap and configure slack as you please without being prompted.

Sample slack install

The files within your slack install

.
|-- etc
|   `-- roles.conf
`-- roles
    `-- web
        `-- files
        |   |-- etc
        |   |   |-- apache2
        |   |   |   |-- sites-available
        |   |   |   |   |-- project.com
        `-- scripts
            |-- postinstall
            |-- preinstall

slack@bb1:~/project$ cat etc/roles.conf 
debian: web

slack@bb1:~/project$ cat roles/web/scripts/preinstall 
#!/bin/bash

echo 'Starting installation' | wall

# install packages
apt-get -q -y --force-yes install subversion git-core apache2 memcached exim4 \
php-codesniffer php-doc php5-imagick php5-memcache libapache2-mod-php5 \
php-pear php5-cli php5-common php5-curl php5-dbg php5-dev php5-gd php5-gmp \
php5-imap php5-mcrypt php5-mhash php5-mysql php5-sqlite php5-tidy php5-xmlrpc \
php5-xsl php5-xdebug php-apc php5-memcached libelastica-php5 php5-zmq \
munin-node munin-plugins-extra mon tripwire pwgen

INSTALLER_LOG=/var/log/non-interactive-installer.log

installnoninteractive(){
  sudo bash -c "DEBIAN_FRONTEND=noninteractive aptitude install -q -y $* >> $INSTALLER_LOG"
}

installnoninteractive mysql-server

# Alternatively you can set the mysql password with debconf-set-selections
MYSQL_PASS=$(pwgen -s 12 1);
mysql -uroot -e "UPDATE mysql.user SET password=PASSWORD('${MYSQL_PASS}') WHERE user='root'; FLUSH PRIVILEGES;";
echo "MySQL Password set to '${MYSQL_PASS}'. Remember to delete ~/.mysql.passwd" | tee ~/.mysql.passwd;


slack@bb1:~/project$ cat roles/web/scripts/postinstall 
#!/bin/bash

echo 'Starting post installation' | wall

# force hostname
hostname -F /etc/hostname

a2ensite project.com

# disable sites
a2dissite default
a2dissite default-ssl

# restart apache
/etc/init.d/apache2 restart

Running

vagrant up
vagrant ssh

More examples of installing MySQL unattended can be found at Installing Java & MySQL – unattended/non-interactive installation

Share

PHP5.3 autoload/class_exists fail

In: PHP

10 Dec 2011

Every now and then I come across a problem with PHPs autoloading whereby class_exists fails to autoload classes. I’ve noticed this with a number of PHP versions, most recently with 5.3.2-1ubuntu4.9. Typically I’ve resolved this by simply upgrading the PHP version.

In the following example the initial class_exists fails until you instantiate the object and from that point the class exists.

PHP 5.3.2-1ubuntu4.9 with Suhosin-Patch (cli) (built: May 3 2011 00:43:34)
Copyright (c) 1997-2009 The PHP Group
Zend Engine v2.3.0, Copyright (c) 1998-2010 Zend Technologies
with Xdebug v2.1.0, Copyright (c) 2002-2010, by Derick Rethans

This example is simply using Zend Framework + Doctrines Annotations.

var_dump(class_exists('\Everlution\Annotations\Acl', true)); // bool(false)
$obj = new \Everlution\Annotations\Acl(array());
var_dump($obj);
var_dump(class_exists('\Everlution\Annotations\Acl', true)); // bool(true)

bool(false)
object(Everlution\Annotations\Acl)#160 (1) {
  ["_values":protected]=>
  array(0) {
  }
}
bool(true)

Share

Selenium

In: General|PHP

1 Dec 2011

A little while ago, I did an internal presentation on selenium at Everlution. I’ve put this on slideshare, which can be found here.

The key problem to entry is simply getting people to record the actions and the issues they are reporting. I often find trying to reproduce the problem on complex software can take around 30 minutes. With a simple recording using selenium IDE, you can easily replay and help isolate the problem. Not all of our clients are onsite, as such this takes the developer longer to identify or clarify what the issue is. Often tickets provide many ambiguous interpretations and the mean time to resolving an issue would be reduced.

By building a collection of these tests with any issues that do arise you will be building a larger collection of repeatable tests and the key to this is repeatability.

Selenium

Some of there references I used can be found at:

Share

Installing Java & MySQL – unattended/non-interactive installation

In: General|Linux|MySQL

1 Dec 2011

This is something I commonly have to run and always seem to forget. In order to run unattended.

Java 6 installation on Ubuntu

add-apt-repository "deb http://archive.canonical.com/ $(lsb_release -s -c) partner"
apt-get update

echo "sun-java6-jdk shared/accepted-sun-dlj-v1-1 select true" | debconf-set-selections
echo "sun-java6-jre shared/accepted-sun-dlj-v1-1 select true" | debconf-set-selections

DEBIAN_FRONTEND=noninteractive aptitude install -y -f sun-java6-jre sun-java6-bin sun-java6-jdk

Java 6 installation on Debian

# Simply add non-free on your main sources list

cat > /etc/apt/sources.list.d/bytemark.list <<-EOD
deb http://mirror.bytemark.co.uk/debian/ squeeze main contrib non-free
deb-src http://mirror.bytemark.co.uk/debian/ squeeze main contrib non-free

deb http://security.debian.org/ squeeze/updates main contrib non-free
deb-src http://security.debian.org/ squeeze/updates main contrib non-free

deb http://mirror.bytemark.co.uk/debian/ squeeze-updates main contrib non-free
deb-src http://mirror.bytemark.co.uk/debian/ squeeze-updates main contrib non-free
EOD

echo "sun-java6-jdk shared/accepted-sun-dlj-v1-1 select true" | debconf-set-selections
echo "sun-java6-jre shared/accepted-sun-dlj-v1-1 select true" | debconf-set-selections

apt-get update
DEBIAN_FRONTEND=noninteractive aptitude install -y -f sun-java6-jre sun-java6-bin sun-java6-jdk

MySQL installation on Debian/Ubuntu

Will install mysql without a password

#!/bin/bash
INSTALLER_LOG=/var/log/non-interactive-installer.log

installnoninteractive(){
  sudo bash -c "DEBIAN_FRONTEND=noninteractive aptitude install -q -y $* >> $INSTALLER_LOG"
}

installnoninteractive mysql-server

# Alternatively you can set the mysql password with debconf-set-selections
apt-get install -f -y pwgen >/dev/null;
MYSQL_PASS=$(pwgen -s 12 1);
mysql -uroot -e "UPDATE mysql.user SET password=PASSWORD('${MYSQL_PASS}') WHERE user='root'; FLUSH PRIVILEGES;";
echo "MySQL Password set to '${MYSQL_PASS}'. Remember to delete ~/.mysql.passwd" | tee ~/.mysql.passwd;

MySQL installation on Debian/Ubuntu

apt-get install -f -y pwgen >/dev/null;
MYSQL_PASS=$(pwgen -s 12 1);

cat <<MYSQL_PRESEED | debconf-set-selections
mysql-server-5.1 mysql-server/root_password password $MYSQL_PASS
mysql-server-5.1 mysql-server/root_password_again password $MYSQL_PASS
mysql-server-5.1 mysql-server/start_on_boot boolean true
MYSQL_PRESEED

sudo DEBIAN_FRONTEND=noninteractive apt-get install -f -y mysql-server

echo "MySQL Password set to '${MYSQL_PASS}'. Remember to delete ~/.mysql.passwd" | tee ~/.mysql.passwd;

Additional notes after install:

# Place on a public interface
sudo sed -i 's/127.0.0.1/0.0.0.0/g' /etc/mysql/my.cnf
sudo service mysql restart

Reference:

scripted installation java ubuntu

Share
Older Entries

About this blog

Andrew Johnstone is a software engineer / lead developer working at Everlution Software.

Photostream

Categories

Archives

  • Steve: Hi there... sorry that this is old, but I'm trying to use your script to check for my usage per seco [...]
  • vinodh: great. thanks . My boss wanted me to configure multiple mysql instances on same physical machine . i [...]
  • andrew.johnstone: In the example above this was in fact using spl_autoload_register. I've never debugged this properly [...]
  • Mike: I had a similar problem I think where a random class would fail to load when others loaded just fine [...]
  • John Cannon: Hey that was really needful. Thanks for sharing. I'll surely be looking for more. [...]

GitHub

GitHub Octocat

ajohnstone @ GitHub

  • Status updating...

Copyright © 2007 - Development, Analysis And Research - is proudly powered by WordPress | Log in

Compositio Theme is created by: Design Disease brought to you by PremiumThemes.com