mtak.nl

Night mode on iPhone

2017-09-22T00:00:00+02:00

A colleague of mine was so impressed by this, I had to make a post about this.

When you’re in bed, often the brightness on your iPhone is still too high. Mostly this can be solved by switching apps to night mode, but not all apps or websites support this feature. So, what you could do, is use one of the built-in accessibility features of iPhone, which allows you to invert the colors. This essentially turns your screen into a photo negative. It can even be bound to a hotkey (three clicks of the home button), after which your screen will look like this:

To set this up, do the following:

Open the Settings app and go to General->Accessibility
At the bottom of this menu, select Accessibility shortcut

Set this option to Invert Colours

When you tap the home button three times in quick succession, you will see the following change:

It does make images look weird, but for some late-night reading, it should be good enough.

Check out my colleague’s website at Brasant.com

ESP8266-based power meter in Kibana

2016-07-21T00:00:00+02:00

I was looking for an easy way to measure my power usage and visualize it in a nice way. Using existing components in my setup and some cheap hardware I came up with something thats easy to build and reliable.

I am using an ESP8266-based board, the Wemos D1 Mini. If you’re not familiar with the ESP8266 line of products, in short it’s an Arduino with built-in WiFi for around USD 3. As you can see in the picture, I have connected it to an LDR and an MQTT message bus. The LDR measures the pulses coming from my utility company’s power meter and converts it to Watts usage. Then the data goes via an MQTT bus with some rewriting in Node-RED to an Elasticsearch server, which allows me to visualize the data using Kibana.

Wemos D1 Mini with LDR

The (WeMos D1 Mini)[http://www.wemos.cc/Products/d1_mini.html] is an ESP8266 breakout. It’s relatively small and cheap on AliExpress. I used the Arduino library to program the ESP8266 using Arduino-style code.

#include <Arduino.h>
#include <ESP8266WiFi.h>
#include <PubSubClient.h>
#include "DHT.h"
#include <TM1637Display.h>

#define DHTPIN D
#define DHTTYPE DHT11

#define SEGCLK 2
#define SEGDIO 3

const char* ssid = "xxx";
const char* password = "yyy";
const char* mqtt_server = "mqtt.int.mtak.nl";

const char* dhtTopic = "meterkast/dht";
const char* ldrTopic = "meterkast/power";

WiFiClient espClient;
PubSubClient client(espClient);
DHT dht(DHTPIN, DHTTYPE);
TM1637Display display(SEGCLK, SEGDIO);

long lastDht = 0;
char msg[50];
long lastLdrTime = 0;
long lastLdrUpdate = 0;
long lastDisplayUpdate = 0;
int lastDisplayState = 0;
int lastPower = 0;
int lastTemp = 0;

void setup() {
  pinMode(BUILTIN_LED, OUTPUT);     // Initialize the BUILTIN_LED pin as an output
  digitalWrite(BUILTIN_LED ,0);     // Turn LED on while starting

  uint8_t data[] = { 0xff, 0xff, 0xff, 0xff };
  display.setBrightness(0x0f);
  display.setSegments(data);
  
  Serial.begin(115200);
  setup_wifi();
  client.setServer(mqtt_server, 1883);
  client.setCallback(callback);

  dht.begin();
  digitalWrite(BUILTIN_LED ,HIGH);     // Turn LED off
}

void setup_wifi() {

  delay(10);
  // We start by connecting to a WiFi network
  Serial.println();
  Serial.print("Connecting to ");
  Serial.println(ssid);

  WiFi.begin(ssid, password);

  while (WiFi.status() != WL_CONNECTED) {
    delay(500);
    Serial.print(".");
  }

  Serial.println("");
  Serial.println("WiFi connected");
  Serial.println("IP address: ");
  Serial.println(WiFi.localIP());
}

void callback(char* topic, byte* payload, unsigned int length) {
  Serial.print("Message arrived [");
  Serial.print(topic);
  Serial.print("] ");
  for (int i = 0; i < length; i++) {
    Serial.print((char)payload[i]);
  }
  Serial.println();

}

void reconnect() {
  // Loop until we're reconnected
  while (!client.connected()) {
    Serial.print("Attempting MQTT connection...");
    // Attempt to connect
    if (client.connect("client_identifier", "mqtt_username", "mqtt_password")) {
      Serial.println("connected");
    } else {
      Serial.print("failed, rc=");
      Serial.print(client.state());
      Serial.println(" try again in 5 seconds");
      // Wait 5 seconds before retrying
      delay(5000);
    }
  }
}

void sendDhtSensor() {
  float h = dht.readHumidity();
  float t = dht.readTemperature();

  if (isnan(h) || isnan(t)) {
    Serial.println("Failed to read from DHT sensor!");
    client.publish("dhtTopic", "Failed to read from DHT sensor");
    return;
  }

  char dhtMessage[120];
  sprintf(dhtMessage, "{ \"temperature\": %d, \"humidity\": %d }", round(t), round(h));
  lastTemp = round(t);

  Serial.print("Publish message: ");
  Serial.println(dhtMessage);
  
  client.publish(dhtTopic, dhtMessage);
}

void sendPower(long ldrTime) {
  float watts;

  watts = 3600000 / ldrTime;
  
  char ldrMessage[120];
  sprintf(ldrMessage, "{ \"power\": %d }", round(watts) );
  lastPower = round(watts);
  
  Serial.print("Publish message: ");
  Serial.println(ldrMessage);
  
  client.publish(ldrTopic, ldrMessage);
}
void loop() {

  if (!client.connected()) {
    reconnect();
  }
  client.loop();

  long now = millis();

  // Send DHT sensor data every 60s
  if (now - lastDht > 60000) {
    lastDht = now;
    sendDhtSensor();
  }

  // Only read every 20ms
  //currentTime = millis();
  if ( lastLdrUpdate < ( now - 20 ) ) {
    lastLdrUpdate = now;
    
    // Send power data when necessary (threshold = 200)
    long ldrValue;
    ldrValue = analogRead(A0);
    if ( ldrValue > 200 ) {
      long ldrNow = millis();
    
      long ldrTime;
      ldrTime = ldrNow - lastLdrTime;
    
      // Skip double detections
      if ( ldrTime < 500 ) { return; }
    
      lastLdrTime = ldrNow;
      sendPower(ldrTime);
    }

  }  // lastLdrUpdate < 20ms 
}

tldr; There are only a few values to change. Update the WiFi SSID, password and MQTT server name around line 13 and the MQTT username and password around line 97.

The script sets up a connection over WiFi using MQTT to the Mosquitto server. It then polls the DHT sensor every 60 seconds and sends the temperature and humidity to the MQTT server. It reads the value of the LDR every 20 ms, and if the value is higher than the threshold, calculate the wattage based on the time difference between the last pulse and the current pulse. It then publishes this value to MQTT as well.

Node-RED

Node-RED is setup to receive the data published to the topics the above Arduino code uses. It then creates a JSON object from the data, adds a timestamp for publishing to ElasticSearch and uploads the data to ES using an HTTP POST request.

Add Timestamp block

The date is published twice in the messages, which seems odd. The es_timestamp value is used as the @timestamp key in ES. The index_date value is used for auto-generating the ES index name (sensors-2016-07-25 etc) as you can see in the HTTP Block.

d = new Date();

function ISODateString(d){
    function pad(n){return n<10 ? '0'+n : n}
    return d.getUTCFullYear()+'-'
    + pad(d.getUTCMonth()+1)+'-'
    + pad(d.getUTCDate())+'T'
    + pad(d.getUTCHours())+':'
    + pad(d.getUTCMinutes())+':'
    + pad(d.getUTCSeconds()) + '.000Z'
}

function DateString(d){
    function pad(n){return n<10 ? '0'+n : n}
    return d.getUTCFullYear()+'.'
    + pad(d.getUTCMonth()+1)+'.'
    + pad(d.getUTCDate())
}

msg.payload.es_timestamp = ISODateString(d);
msg.index_date = DateString(d);
return msg;

Template block

This block generates the POST data that will be sent to Elasticsearch.

{ "@timestamp":"{{payload.es_timestamp}};", "power":{{payload.power}} }

HTTP Block

Check Elasticsearch

You can check if there is data being added to Elasticsearch with:

$ curl -XGET 'http://elasticsearch:9200/sensors-*/_search?q=_type:meterkast-power&size=20&sort=@timestamp:desc'

This query will display the last 20 entries for the power index and type.

Kibana

Add the index to Kibana (you can use the wildcard sensors-* form). A few examples of graphs you could make:

Gnuplot

To render images to websites, you could use the data from ES to make graphs using Gnuplot. For example, you could get the data for the last 24 hours from ES:

curl -XGET 'http://elk1.int.mtak.nl:9200/sensors-*/_search?pretty' -d '
{
  "query": {
    "filtered": {
      "query": {
        "query_string": {
          "analyze_wildcard": true,
          "query": "_type: \"meterkast-power\""
        }
      },
      "filter": {
        "bool": {
          "must": [
            {
              "query": {
                "query_string": {
                  "analyze_wildcard": true,
                  "query": "*"
                }
              }
            },
            {
              "range": {
                "@timestamp": {
                  "gte": "now-24h"
                }
              }
            }
          ],
          "must_not": []
        }
      }
    }
  },
  "size": 9999
}
' 2>/dev/null | egrep 'power|timestamp' | egrep -v 'type' | awk '{print $3}' | sed -e 'N;s/,\n/, /' -e 's/"//g'

And then plot this data using Gnuplot:

set term png size 512,181 enhanced font 'Verdana,7'
set style data fsteps
set output '/var/www/tools/api/sensor/power-24h.png'
set xdata time
set timefmt '%Y-%m-%dT%H:%M:%S.000Z'
set format x "%H"
set datafile sep ','
set title "Meterkast-Power - Last 24 hours (UTC)" font 'Verdana,9'
plot 'power-24h.dat' using 1:2 title 'Watts' with lines lt rgb "#00DD00"

You get a graph that will look like:

Keyboard optimization

2016-06-04T00:00:00+02:00

In this post I get a bit more philisophical than technical. IT engineer’s no. 1 tool: the keyboard. A often overlooked peripheral, but a few optimizations can bring you a lot of joy.

Self-confident and trained keyboard usage can speed up your progress in projects dramatically. Here are a few tips to improve your keyboard life and get more joy out of typing.

Keyboard layout

Some people radically change keyboard layouts. I know Qwerty isn’t optimal, but Dvorak is just one bridge too far for me. There are some tweaks which make things easier in my daily life.

Disable internationalization/dead keys! Dead keys are the worst if you’re doing any kind of programming or shell work. Every time you type a \, ' or " you have to press the spacebar to get the actual character, or hope that the next character you’re typing isn’t something that combines with one of these modifiers. So what about if I need a é, ï or å? You can use the old-skool Compose key, it’s a bit more work, but you don’t have to deal with dead keys anymore. This is how:
- In Ubuntu, enable compose key by going to System Settings -> Keyboard -> Shortcuts -> Typing and set it to a key (I use Right Alt). How do you use it?
- Press the Compose Key, then type the modifier key (', ", etc.) and then type the character that should be modified. Do not press the keys at the same time, press them sequentially.
- Example: Right Alt -> " -> e gives ë
- Example: Right Alt -> , -> c gives ç
Ctrl-Caps Lock swap Old Unix machines could be ordered with a Unix style keyboard. What’s the difference with a normal keyboard? They had the Ctrl and Caps Lock keys switched. Proponents say it helps with pinky aches, because you don’t have to reach as far to hit Ctrl, which the average Unix guy does often. It’s a bit annoying to get used to though. What I did is change Caps Lock to Ctrl, leave Ctrl as it is and press both shift buttons at the same time to turn on Caps Lock. I hardly use Caps Lock anyways, and I can still use it if I need to.
- There are several ways to do this. The easiest is to install gnome-tweak-tool.
- Under Typing I set the following options:
  - Miscellaneous compatibility options: Both shift keys together toggle caps lock
  - Ctrl key position: Caps Lock as Ctrl

Mechanical keyboards

I’m not saving the best ‘till last. A good-quality mechanical keyboard is so much more enjoyable to type on. It takes less effort to press keys, and it will give you tactile (or audible) feedback when you press a key. This usually results in a faster typing speed. If you typed on a mechanical keyboard and go back to rubber-dome keyboards, it’ll feel like typing on a sponge!

Lots of guides have been written about mechanical keyboards. An excellent resource is the Mechanical keyboard guide at overclock.net. There are some thing you ought to consider:

When working in a office environment, please, please, please consider your colleagues and get keyboard switches with tactile but not audible feedback. Even with Cherry MX Brown switches, “bottoming out” (hitting the keyboard backplate when you type too hard) really get’s on people’s nerves after a while. I recommend getting rubber O-rings for additional dampening.
Think about getting a tenkeyless keyboard. Tenkeyless keyboards do not have the num pad section like most keyboards do. Why? Most engineers don’t type numbers that much, but it does get your keyboard more directly in front of you, limiting the twisting in your wrists. It will also be easier to switch between keyboard and mouse, because the gap between your main typing area (letters) and the mouse is less. I have used a tenkeyless keyboard for years now and I hardly miss the num pad. It does seem a bit odd that you pay as much for a tenkeyless keyboard as you do for a normal one.
Pimp your keyboard Pump up your nerd-cred with customized keys. A lot of vendors on the internet sell keycaps for Cherry MX switches, you can even get custom ones made. I have “ninja” keys (the lettering is on the front of the key instead of the top), a red ESC key and Sun Microsystems keys instead of Windows keys. You can definitely tell it’s my keyboard, because my username is on the space bar.

If you opt to go for a mechanical keyboard, make sure you pick the right keyboard layout. Nothing is more annoying that being used to typing on an ANSI keyboard and getting an ISO layout.

Images

Resource friendly Git server

2016-06-03T00:00:00+02:00

Keeping in line with my policy of low memory overhead, I do not want to waste several hundred megabytes on a git repository manager, such as Gitlab or Bitbucket. If you get back to Git basics, you can set up a Git repository with no additional overhead.

Basics

Let’s get back to basics, what is Git? Git is a distributed version control system, which means it’s just a bunch of files on disk. Git is fairly protocol-agnostic, so it can transfer data over HTTP, FTP or the Git protocol. Often, the Git protocol is used in conjunction with ssh, which handles the transport layer encryption and authentication. Whenever you see a command line git clone git@git:repo, you are most likely using Git+SSH.

This guide assumes you know how SSH public-private key authentication works. If not, follow this tutorial on Digital Ocean.

Setup

Git+SSH

To set up a Git+SSH server, so you can clone/commit etc. via SSH is fairly simple. Create a single Unix user, through which all users will authenticate based on their password. To make it a bit more user-friendly and secure, do the following steps:

Create a Git user.
```
groupadd git
useradd -g git -m -d /srv/git -s /usr/bin/git-shell -c "Git User" git
```
There are several aspects which are important in this command:
- -m -d /srv/git This creates a home directory for the git user which will store all the repositories. Since you can use relative paths with SSH, this allows you to put a repository in /srv/git/repo1 and clone it with git clone git@git.example.com:repo1. No need to use full paths.
- -s /usr/bin/git-shell is a purpose built shell for Git use only. Remember, SSH normally grants you a login shell, which allows you to do everything on a system what a non-root user can, which is a lot! The git-shell only allows clients to execute Git related commands.
Tip: You might need to add /usr/bin/git-shell to the list of allowed shells (often found at /etc/shells) to allow logins using git-shell. This depends on your distribution.
Make sure the git user home directory is owned by the git user:
```
chown git: /srv/git
```
Add public keys for the users to the git user’s authorized_keys file:
```
mkdir /srv/git/.ssh
chmod 700 /srv/git/.ssh
echo 'ssh-rsa AAAAB3NzaC1yc2EAA...ZGXMchiG0K4aNp5= mtak@mtak.nl' >> /srv/git/.ssh/authorized_keys
chmod 600 /srv/git/.ssh/authorized_keys
chown -R git: /srv/git/.ssh
```
Make sure you end up with the following permissions:
```
drwx------ 2 git git 4096 nov 23  2015 .ssh
-rw------- 1 git git 218  nov 23  2015 .ssh/authorized_keys
```
Tip: if you have problems logging on using Git, test if you can log on with a normal SSH client. Make sure the permissions for the .ssh/ directory and authorized_keys file exactly match the ones stated above.
Create a Git repository on the server. You need to create a ‘bare’ Git repository to allow operations over SSH.
```
cd /srv/git
git init --bare repoName
chown -R git: repoName
```
Clone and work with the repo!

You can now work with the repository by using the git remote git@git.example.com:repoName, for example with:

git clone git@git.example.com:repoName
git remote add origin git@git.example.com:repoName

HTTP Browsing

So now we have a basic Git server running, we can enhance it a bit by allowing read-only HTTP browsing of the repositories for easy access. Since I already have an Apache HTTP server with PHP running, I decided to use GitList. The stable GitList version at time of writing is 0.5.0.

Download GitList

wget https://s3.amazonaws.com/gitlist/gitlist-0.5.0.tar.gz

Untar it in your webroot or other directory which is accessible by your webserver
```
cd /var/www/
tar -zxvf /home/user/Downloads/gitlist-0.5.0.tar.gz
```

Copy the config.ini-example file to config.ini and change the following parameters:

[git]
repositories[] = '/srv/git';

[date]
timezone = 'Europe/Amsterdam'

Make sure the Git repository directory is readable by the webserver user. The easiest way to do this is by adding the webserver user to the git Unix group:
```
usermod -G git www-data
```
Browse to the GitList page (probably at http://git.example.com/gitlist/)

Optionally, add authentication and SSL. Use this Apache config as a guide:

<VirtualHost 192.0.2.10:443>
  ServerAdmin admin@git.example.com
  ServerName git.example.com
   
  DocumentRoot /var/www/gitlist/
   
  SSLEngine on
  SSLProtocol all -SSLv2
  SSLCipherSuite ALL:!ADH:!EXPORT:!SSLv2:RC4+RSA:+HIGH:+MEDIUM
   
  SSLCertificateFile /etc/apache2/ssl/git.example.com.crt
  SSLCertificateKeyFile /etc/apache2/ssl/git.example.com.key
  SSLCACertificateFile /etc/apache2/ssl/git.example.com_ca.crt
   
  ErrorLog ${APACHE_LOG_DIR}/git.example.com_error.log
  CustomLog ${APACHE_LOG_DIR}/git.example.com_access.log combined
  LogLevel warn
   
  <Directory /var/www/gitlist>
    Options MultiViews FollowSymLinks
    AllowOverride all
    Order allow,deny
    allow from all
   
    AuthName "GitList"
    AuthType Basic
    AuthBasicProvider ldap
    AuthLDAPURL ldap://ldap.example.com/dc=example,dc=com?uid
    AuthLDAPGroupAttribute memberUid
    AuthLDAPGroupAttributeIsDN off
    Require ldap-group cn=git,ou=Group,dc=example,dc=com
  </Directory>
</VirtualHost>
# Upstep to SSL
<Virtualhost *:80>
  ServerAdmin admin@mtak.nl
  ServerName git.example.com

  Redirect 301 / https://git.example.com/
</VirtualHost>

Read-only HTTP clones

As a mechanism to deploy software, it might come in handy to have read-only access to the repository for certain servers. I use this mechanism to deploy Puppet code accross the network.

Set up Apache HTTPd with the appropriate config:

  Alias "/puppet-readonly.git" "/srv/git/puppet.git/"
  <Directory /srv/git/puppet.git/>
    AuthName "Puppet git repository"
    AuthType Basic
    AuthUserFile /etc/apache2/puppet.git.htpasswd
    Require valid-user
  </Directory>

Create a .htpasswd file for authentication

htpasswd -c /etc/apache2/puppet.git.htpasswd user1

When Git clones or pulls a repository over SSH, it discovers references and packs dynamically. This cannot be done over HTTP, so you need to make sure a list up these is updated every time you push changes to the repository. This can be automated by using Git hooks. Uncomment the exec git update-server-info line in the repository’s hooks/post-update.sample file and rename it to hooks/post-update
```
less /srv/git/puppet/hooks/post-update
#!/bin/sh
#
# An example hook script to prepare a packed repository for use over
# dumb transports.
#
# To enable this hook, rename this file to "post-update".

exec git update-server-info
```
Push a change to the repository to check if the Git hook works. update-server-info should change the following files in the repository:
```
objects/info/packs
info/refs
```

Access the Git repository through the following url:

git clone https://user1:password@git.example.com/puppet-readonly.git

Stand-alone Puppet

2016-06-02T00:00:00+02:00

In my home infrastructure I am running as much services on a single machine in order to save power. The most constrained resource is memory, so I try to save memory wherever I can. Using a Puppetmaster server to serve 10-15 nodes seems a waste of resources, so that’s why I decided to go with a stand-alone setup.

Running Puppet in a stand-alone setup might not be as daunting as you might think. Documentation and examples are sparse across the Internet, so I would like to show you how I did my setup. The only prerequisite for this configuration is that you have a working Git server. Puppet will be automatically executed using cron.

Overview

Puppet will run in stand-alone mode on all machines. Puppet modules and related files will al be kept in a Git repository, so changes can be deployed quickly across the network. Cron will periodically do a git pull and run Puppet, to ensure compliance and quick rollout. This guide assumes you are familiar with the basics of Linux, Git and Puppet.

Basic Puppet setup

The puppet/ directory will contain all files Puppet needs to run on all systems. This includes the code, what to run where and the configuration of the modules.

Create a repository for use with Puppet
Create the following directories:
- manifests/
- modules/

Create a Puppet config file.

puppet.conf:

[main]
logdir=/var/log/puppet
vardir=/var/lib/puppet
ssldir=/var/lib/puppet/ssl
rundir=/var/run/puppet
factpath=$confdir/facter

Create a ‘site’ manifest file. Clients will use this file to determine which modules to run.

manifests/site.pp:
```
node default {
  include ntp
}
```
This will include the NTP module. Feel free to add any more modules with additional include statements. The default node will be used by all clients which don’t have a specific entry in the site.pp file. If you want to change the list of modules for a specific client, you can add multiple node blocks:
```
node default {
  include ntp
}
node mail1 {
  include ntp
  include postfix
}
```
This is not the recommended way to add modules to a host. It is much more convenient to use a intermediate module or use a pattern such as the role-profile pattern.
Add modules to the modules/ directory. In this example we need to put the NTP module in this directory. Download the module from Puppetforge and un-tar it in the modules/ directory.
Commit and check in to the Git repository.

This should give you enough to get a basic setup running. Install Puppet from your favourite repository manager and run it with:

# /usr/bin/puppet apply /etc/puppet/manifests/site.pp

Tip: You will have to debug any errors before proceeding to the next step.

Cron setup

Cron will be used to periodically pull the repo for any changes and run Puppet if anything changed. Puppet will also run periodically (at a lower rate) to ensure system compliance. Now that we have Puppet running, we might as well use it to make the cron config. Time for our first custom module!

We’ll perform the following functions with Puppet:

Make sure Git is installed
Disable the Puppet daemon (we’re running from cron instead)
Add a git hook to the repository to run Puppet when a change is pulled
Add a cron task and schedule it with some randomness to not distribute the load on the Git/virtualization server

In this example we assume you clone the Git repo to /etc/puppet on the clients.

Create the directory structure for the new module:

mkdir modules/cron-puppet
mkdir modules/cron-puppet/manifests
mkdir modules/cron-puppet/templates
mkdir -p modules/cron-puppet/lib/puppet/parser/functions/

Create the template files:

modules/cron-puppet/templates/cron.erb:

# This file is managed by Puppet (cron-puppet)

# Do a Git pull every 10 minutes
<%= @rand_git_pull %> * * * * root /usr/bin/git -C /etc/puppet pull >/dev/null 2>&1

# Do a Puppet run anyways every hour
<%= @rand_minute %> * * * * root /usr/bin/puppet apply /etc/puppet/manifests/site.pp >/dev/null 2>&1

modules/cron-puppet/templates/post-merge.erb:

#!/bin/bash -e
# This file is managed by Puppet (cron-puppet)
/usr/bin/puppet apply /etc/puppet/manifests/site.pp

Tip: It is always good practice to include the name of the Puppet class that is responsible for a file in the template. In that case, it’s easy to find the corresponding class when looking at a file on the system

Add the manifest files:

modules/cron-puppet/manifests/init.pp:

class cron-puppet {
   
  include cron-puppet::last-run
   
  package { 'git':
    ensure => 'latest',
  }
   
  # Disable puppet daemon as it only works with a puppetmaster
  service { 'puppet':
    ensure  => stopped,
    enable  => false,
    require => Package['puppet'],
  }
   
  # Git hook to run puppet when the repo changes
  file { 'post-hook':
    ensure  => file,
    path    => '/etc/puppet/.git/hooks/post-merge',
    content => template('cron-puppet/post-merge.erb'),
    mode    => 0755,
    owner   => root,
    group   => root,
    require => Package['puppet', 'git'],
  }
   
  $rand_minute = fqdn_rand(60)
  $rand_git_pull = generateCronTime()
  file { 'puppet-cron':
    ensure  => file,
    path    => '/etc/cron.d/puppet',
    content => template('cron-puppet/cron.erb'),
    mode    => 0644,
    owner   => root,
    group   => root,
  }
}

The manifest is reasonably self-explanatory. The fqdn_rand() function comes from the Puppet parser, it gives a random number, max 60, based on the FQDN of the host. The generateCronTime() function generates an array of minutes, every 5 minutes, based on this value (so if $rand_minute = 2 then $rand_git_pull = [2,7,12,17...]. This is done so we can schedule a git pull every 5 minutes.

The cron-puppet::last_run module will write the time the Puppet agent last ran to a file, for monitoring purposes.

modules/cron-puppet/manifests/last-run.pp:

class cron-puppet::last-run {
  include stdlib
   
  $time = strftime("%Y-%m-%d %H:%M:%S")
  $timestring = "$time\n"
   
  file { '/run/puppet/last_run.txt':
    ensure    => file,
    owner     => 'root',
    group     => 'root',
    mode      => 0444,
    content   => $timestring,
  }
   
}

Add the extra function we need to generate the cron scheduling times.

modules/cron-puppet/lib/puppet/parser/functions/generateCronTime.rb:

require 'digest/md5'
   
module Puppet::Parser::Functions
  newfunction(:generateCronTime, :type => :rvalue) do |args|
     
    minuteArray = Array.new
    # Generate random offset (from Puppet fqdn_rand.rb)
    seed = Digest::MD5.hexdigest([self['::fqdn'],args].join(':')).hex
    max=10
   
    if defined?(Random) == 'constant' && Random.class == Class
      offset = Random.new(seed).rand(max).to_s
    else
      srand(seed)
      offset = rand(max).to_s
      srand()
      offset
    end
   
    # Generate the 5 minute increments with the offset
    (0..50).step(10) do |n|
      minuteArray.push(n.to_i + offset.to_i)
    end
   
    minuteString=minuteArray.join(",")
    return minuteString.to_s
  end
end

Add the cron-puppet module to the site manifest:

manifests/site.pp:

node default {
  include ntp
  include cron-puppet
}

Run Puppet on the clients:

/usr/bin/puppet apply /etc/puppet/manifests/site.pp

This will apply the cron configuration and Puppet should run automatically now! The last run time can be checked by viewing the /run/puppet/last_run.txt file.

Also check other articles tagged puppet for more tutorials and articles regarding Puppet.

Caveats

For optimal security, clients should have read-only access to the Git repository. This can be done in several ways:
- If you use Git on a file system with ssh access, you could create a different user for access to the repository and use filesystem access controls to make the repo read-only for clients.
- If you serve Git through HTTP, the contents will be read-only automatically (unless you have WebDAV enabled ofcourse :wink: ) This is described at Resource friendly Git server
- Use access controls on your favorite Git repository tooling (Gitlab/Bitbucket)
If you or Puppet somehow manage to break the cron config running the Puppet client, you will have to manually fix it on all nodes.
There is no reporting on the Puppet runs on the network, so you won’t be able to see if all nodes have converged properly. I modified the script which takes the Puppet output status and POST’s it to a webserver, which stores it in a database.

Vyatta LookingGlass

2013-04-23T00:00:00+02:00

Lately I’ve been involved in the dn42 darknet initiative. It’s purpose is to let people build an internet on top of the internet by using VPN tunnels and BGP routing.

In order to aid troubleshooting and insight in the network, I’ve been looking for a BGP looking glass for Vyatta, the routing software I use. I couldn’t find it, so I took it upon myself to build one.

It’s a web frontend which drives a Perl backend using AJAX. It connects to your router with SSH, and returns the output to the client. For all information, view the README file.

Download from GitHub GitHub Repository

Using SPICE with QEMU/KVM on Ubuntu 12.04

2013-04-15T00:00:00+02:00

SPICE is a new remote desktop protocol, which is being used a lot in combination with the QEMU emulator on KVM. One of the biggest advantages of SPICE over VNC or RDP is that it works at near-native speeds, you can watch full screen video without any delay! SPICE is being developed by Red Hat and a such is supported best on RHEL-like distro’s.

However, I run Ubuntu on my laptop. For my university studies I need to use a Windows VM a lot, so I also wanted the near-native performance of SPICE. Unfortunately Ubuntu’s package for SPICE/QEMU/KVM leaves some to be desired. This is a description of how I set up KVM on my box:

Preparation

Install qemu and spice:

$ sudo apt-get install spice-client-gtk python-spice-client-gtk qemu-kvm-spice virt-manager virt-viewer kvm libvirt-bin

Add your useraccount to the libvirtd group:

$ sudo adduser mtak libvirtd

Create a new VM using the virt-manager tool (Virtual Machine Manager from Unity).
Install your 32-bit Windows OS in VNC mode.
Install the SPICE guest tools on the Windows OS. You can download these from here.

Turn on SPICE

Change the emulator option in the VM config to /usr/bin/kvm-spice using:

$ virsh edit vmname

Reconfigure the VM’s display mode to SPICE mode using the virt-manager tool.
Reboot the Windows VM.
Switch to qlx video mode using the virt-manager tool or the virsh edit command.

Keep in mind that SPICE will only start working after the drivers have been loaded. This means that the screen will stay black during booting and that you will only see something when the system boots up to the login screen. If you need to do some maintenance on your Windows VM, switch back to VNC and Cirrus video adapter.

Simple SMTP AUTH on Postfix

2013-04-10T00:00:00+02:00

Ever since I got an iPhone I wanted to send email through my own mail server. To do this without making an open relay you can use a technique called SMTP authorization. It allows you to authenticate to your SMTP MTA and send message to the whole wide world. Unauthenticated connections will still only be able to send mail to mydestination, virtual_alias_domains and relay_domains. This guide is written using Debian Squeeze (6.0), but apart from the SASL installation and file paths it should be applicable to most Linux distros.

TLS

Whenever you send a password over the network it ought to be encrypted. Add the following parameters to your main.cf to enable TLS:

# TLS parameters
smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem
smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key
smtpd_use_tls=yes
smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache

Of course you need to make sure the cert_file options are pointing to valid certificates. The default snakeoil certificates should be secure enough for most users, but you can always purchase some CA signed certificates.

You can test TLS using the OpenSSL utility

$ openssl s_client -starttls smtp -crlf -connect 127.0.0.1:25
CONNECTED(00000003)
[output truncated]
SSL handshake has read 1646 bytes and written 354 bytes
---
New, TLSv1/SSLv3, Cipher is DHE-RSA-AES256-SHA
Server public key is 2048 bit
Secure Renegotiation IS supported
Compression: NONE
Expansion: NONE
SSL-Session:
    Protocol  : TLSv1
    Cipher    : DHE-RSA-AES256-SHA
    Session-ID: 17C511889EDC11109C9F71C16F7F70AB9B8868129C4B796F96B2F0A68E581A8E
    Session-ID-ctx:
    Master-Key: E391BF9B8940CD36D5E98BC5FAF05860E86AAE4667D5E8E87081AD4A98A7EA91F770790C053A2ECCA42AD2937AD83F90
    Key-Arg   : None
    Start Time: 1365632805
    Timeout   : 300 (sec)
    Verify return code: 18 (self signed certificate)
---
250 DSN
quit
221 2.0.0 Bye
closed
$

SASL

SMTP does not have any authentication capabilities built-in, but it can be extended using the SASL framework. SASL is described in RFC4422. To keep things simple we will use PAM as an authentication backend. Refer to saslauthd(8) for more information regarding authentication backends.

To set up SASL:

Install all the packages needed for SASL:

$ sudo apt-get install libsasl2-2 sasl2-bin libsasl2-modules

Edit /etc/default/saslauthd. You can leave the default settings but make sure these match:

[...]
START=yes
MECHANISMS="pam"
THREADS=1
[...]
OPTIONS="-c -m /var/spool/postfix/var/run/saslauthd"

Create /etc/postfix/sasl/smtpd.conf:

saslauthd_version: 2
pwcheck_method: saslauthd
mech_list: plain login

Modify /etc/postfix/main.cf:

smtpd_sasl_auth_enable = yes
smtpd_sasl_security_options = noanonymous
smtpd_sasl_local_domain =
broken_sasl_auth_clients = yes
smtpd_recipient_restrictions =
   permit_sasl_authenticated,
   permit_mynetworks,
   check_relay_domains

Add the postfix user to the sasl Unix group:

$ adduser postfix sasl

Start the SASL daemon and restart Postfix:

$ /etc/init.d/saslauthd start
$ /etc/init.d/postfix restart

You can test the SMTP AUTH functionality by creating a base64 hash of your password and using that to log on:

$ printf '\0%s\0%s' 'username' 'password' | openssl base64
AHVzZXJuYW1lAHBhc3N3b3Jk

$ telnet localhost 25
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
220 mail.gallische-dorp.net ESMTP Postfix
EHLO localhost
250-mail.gallische-dorp.net
250-PIPELINING
250-SIZE 20480000
250-VRFY
250-ETRN
250-STARTTLS
250-AUTH PLAIN LOGIN
250-AUTH=PLAIN LOGIN
250-ENHANCEDSTATUSCODES
250-8BITMIME
250 DSN
AUTH PLAIN AHVzZXJuYW1lAHBhc3N3b3Jk
235 2.7.0 Authentication successful
quit
221 2.0.0 Bye
Connection closed by foreign host.

Usenet backups

2013-04-08T00:00:00+02:00

Are you looking for free, unlimited, off-site cloud storage for your backups? Why not try Usenet? You can encrypt your backup files, upload them and download them later if you need them. There are Usenet providers who offer up to 3 years retention, so you should be ok.

A colleague mentioned this idea last week, to encrypt your backups and upload them to Usenet. I took his idea and made a shell script to do exactly just that. The script takes a single file and a newspost name as argument and encrypts, rars, pars and uploads the file to Usenet, where you can download it up to three years later.

Download on GitHub

Have fun!