Extend Python Logging

Simple extension of Python core logging that provides ‘thread-local-aware’ logger. The term ‘MDC’ is inspired from MDC concept in various Java logging libraries.

import logging
import threading

__author__ = 'hoang281283@gmail.com'


class MDCLocal(threading.local):

    def __init__(self):
        self.mdc_dict = {"request_id": ""}

    def set_attribute(self, key, value):
        self.mdc_dict[key] = value

    def remove_attribute(self, key):
        del self.mdc_dic[key]

    def get_attribute(self, key):
        return self.mdc_dict[key]

    def get_mdc_map(self):
        return self.mdc_dict

    def clear(self):
        [self.mdc_dict.update({k: ""}) for k in self.mdc_dict]


MDC_BASIC_FORMAT = '%(asctime)s - %(name)s %(levelname)s - [%(request_id)s] : %(message)s'
MDC_MAP = MDCLocal()


class MDCLogger(logging.Logger):

    def __init__(self, deco_logger):
        self.deco_logger = deco_logger

    def setLevel(self, level):
        self.deco_logger.setLevel(level)

    def debug(self, msg, *args, **kwargs):
        self.deco_logger.debug(msg, extra=MDC_MAP.get_mdc_map(), *args, **kwargs)

    def info(self, msg, *args, **kwargs):
        self.deco_logger.info(msg, extra=MDC_MAP.get_mdc_map(), *args, **kwargs)

    def warning(self, msg, *args, **kwargs):
        self.deco_logger.warning(msg, extra=MDC_MAP.get_mdc_map(), *args, **kwargs)

    def warn(self, msg, *args, **kwargs):
        self.deco_logger.warning(msg, extra=MDC_MAP.get_mdc_map(), *args, **kwargs)

    def error(self, msg, *args, **kwargs):
        self.deco_logger.error(msg, extra=MDC_MAP.get_mdc_map(), *args, **kwargs)

    def exception(self, msg, *args, **kwargs):
        self.deco_logger.exception(msg, extra=MDC_MAP.get_mdc_map(), *args, **kwargs)

    def critical(self, msg, *args, **kwargs):
        self.deco_logger.critical(msg, extra=MDC_MAP.get_mdc_map(), *args, **kwargs)

    def log(self, level, msg, *args, **kwargs):
        self.deco_logger.critical(level, msg, extra=MDC_MAP.get_mdc_map(), *args, **kwargs)


def put_mdc(key, value):
    MDC_MAP.set_attribute(key, value)


def clear_mdc():
    MDC_MAP.clear()


def basic_config(**kwargs):
    if 'format' in kwargs.keys():
        pass
    else:
        kwargs['format'] = MDC_BASIC_FORMAT
    logging.basicConfig(**kwargs)


def get_mdc_logger(name=None):
    if name:
        return MDCLogger(logging.getLogger(name))
    else:
        return MDCLogger(logging.root)

Code testing mdc_logging on multiple threads

import mdc_logging, logging, thread

mdc_logging.basic_config()
logger = mdc_logging.get_mdc_logger("SharedLogger")

def print_log_on_thread(s):
    mdc_logging.put_mdc("request_id", s)
    logger.warn("Blah blah")
    mdc_logging.clear_mdc()

thread.start_new_thread(print_log_on_thread, (123456,))
thread.start_new_thread(print_log_on_thread, (654321,))

Inside Fabric SDK

Screenshots of Fabric SDK decompiled code showing:

• How build ID is generated at compile time
• How build ID and API key is read at run time

Generate buildID at compile time
 

 
Extract build ID from R resource

 
API Key and build ID in API request

Braille Quiz

Solution for the Braille quiz

import java.util.HashMap;
import java.util.Map;

/**
 * @author <a href="hoang281283@gmail.com">Minh Hoang TO</a>
 * @date: 6/30/15
 */
public class BrailleToken {

    public final static Map<String, Character> BRAILLE_ALPHABET_MAPPING = new HashMap<>();

    static {
        //Init the maping from Braille spec. If you are lazy, use reflection to make this mapping
        //BRAILLE_ALPHABET_MAPPING.put(".00000", 'a');
    }

    private String firstRow;

    private String secondRow;

    private String thirdRow;

    public BrailleToken(String firstRow, String secondRow, String thirdRow) {
        this.firstRow = firstRow;
        this.secondRow = secondRow;
        this.thirdRow = thirdRow;
    }

    private String matrixToVector() {
        return firstRow + secondRow + thirdRow;
    }

    public Character toAlphabet() {
        return BRAILLE_ALPHABET_MAPPING.get(matrixToVector());
    }
}

and

import java.util.LinkedList;
import java.util.List;

/**
 * @author <a href="hoang281283@gmail.com">Minh Hoang TO</a>
 * @date: 6/30/15
 */
public class TripleLine {

    private String firstLine;

    private String secondLine;

    private String thirdLine;

    public TripleLine(String firstLine, String secondLine, String thirdLine) {
        this.firstLine = firstLine;
        this.secondLine = secondLine;
        this.thirdLine = thirdLine;
    }

    public List<BrailleToken> getBrailleTokens() {
        int size = Math.min(firstLine.length(), Math.min(secondLine.length(), thirdLine.length()));
        List<BrailleToken> res = new LinkedList<>();

        String[] lines = new String[]{firstLine, secondLine, thirdLine};
        for (int n = 0; n < size; n += 3) {
            StringBuilder[] rowsInBraille = new StringBuilder[]{new StringBuilder(), new StringBuilder(), new StringBuilder()};

            //TODO: Provide sublte handling if invalid letters are encoutered
            for (int i = 0; i < 3; i++)
                for (int j = 0; j < 2; j++) {
                    rowsInBraille[i].append(lines[i].charAt(n + j));
                }

            res.add(new BrailleToken(rowsInBraille[0].toString(), rowsInBraille[1].toString(), rowsInBraille[2].toString()));
        }

        return res;
    }
}

and

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Iterator;
import java.util.LinkedList;

/**
 * @author <a href="hoang281283@gmail.com">Minh Hoang TO</a>
 * @date: 6/30/15
 */
public class BrailleParser implements Iterator<BrailleToken> {

    private LinkedList<BrailleToken> buf = new LinkedList<>();

    private BufferedReader reader;

    public BrailleParser(BufferedReader reader) {
        this.reader = reader;
    }

    @Override
    public boolean hasNext() {
        if (buf.isEmpty()) {
            readMore();
        }
        return buf.isEmpty();
    }

    @Override
    public BrailleToken next() {
        return buf.removeFirst();
    }

    private void readMore() {
        String[] lines = new String[3];
        for (int i = 0; i < 3; i++) {
            String l = null;
            try {
                l = reader.readLine();
            } catch (IOException ioEx) {

            }
            if (l == null) {
                return;
            } else {
                lines[i] = l;
            }
        }

        buf.addAll(new TripleLine(lines[0], lines[1], lines[2]).getBrailleTokens());
    }

    public static void main(String[] args) throws Exception {
        BrailleParser bp = new BrailleParser(new BufferedReader(new InputStreamReader(new FileInputStream(args[0]))));
        StringBuilder message = new StringBuilder();

        while(bp.hasNext()){
            message.append(bp.next().toAlphabet());
        }

        System.out.println(message.toString());
    }
}

NestedFetcher Pattern

My own named NestedFetcher pattern that helps us to have clean and modular Android code in case your application make a call to RESTful endpoint for a list of items, then make subsequent asynchronous call while iterating over those items.

import android.os.AsyncTask;

import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.LinkedList;
import java.util.List;

/**
 * @author <a href="hoang281283@gmail.com">Minh Hoang TO</a>
 * @date: 6/29/15
 */
@SuppressWarnings("unchecked")
public abstract class NestedFetcher<F, S> extends AsyncTask<String, Void, List<F>> {

    @Override
    protected final List<F> doInBackground(String... params) {
        InputStream res = null;
        try {
            HttpURLConnection httpReq = (HttpURLConnection) new URL(params[0]).openConnection();
            httpReq.connect();
            res = httpReq.getInputStream();

            return unmarshall(res);
        } catch (Exception ex) {
            return new LinkedList<>();
        } finally {
            if (res != null) {
                try {
                    res.close();
                } catch (IOException ioEx) {
                }
            }
        }
    }

    @Override
    protected final void onPostExecute(List<F> fs) {
        for (F fgRes : fs) {
            createTask(fgRes).execute(fgRes);
        }
    }

    protected abstract AsyncTask<F, Void, S> createTask(F firstGenRes);

    protected abstract List<F> unmarshall(InputStream in);
}

Carriage Return

“How we could print dynamic text to terminal without moving to new line?”

I got this question from one teammate while he was building a big C++ project and looking at build progress updated on Linux terminal.

The key for this interesting question is to somehow move back the cursor of standard output. In most of programming languages, the standard output does not provide API to control cursor and that means we do not have straightforward solution.

Fortunately, the special character “Carriage Return” allows us to move back to the beginning of most recently printed line, and that is barely enough for the question from my teammate (see my Java code below).

http://en.wikipedia.org/wiki/Carriage_return

/**
 * @author <a href="hoang281283@gmail.com">Minh Hoang TO</a>
 * @date: 5/26/15
 */
public class Progress {

    public static void main(String[] args) {
        StringBuilder spaces = new StringBuilder();
        StringBuilder pipelines = new StringBuilder();
        for (int i = 0; i < 100; i++) {
            spaces.append(' ');
            pipelines.append('|');
        }

        for (int i = 0; i < 101; i++) {
            System.out.print("\r");//Move output cursor to the beginning of line
            System.out.append("In progress:")
                    .append(pipelines, 0, i)
                    .append(spaces, i, 100)
                    .append(" " + i)
                    .append("%")
                    .flush();
            try {
                Thread.sleep(100);
            } catch (InterruptedException iex) {

            }
        }
    }
}

Chrome URLs

One nice feature of Chromium-based browsers (ex: Google Chrome, Coccoc,…) is that we could access internal runtime data via URLs having chrome as scheme, those special URLs are referred to as Chrome URLs.

Getting the whole list of Chrome URLs supported on running Chromium-based browser is as simple as checking the link chrome://chrome-urls

Memory Footprint

Open the link chrome://memory-redirect, and we get per-tab memory consumption.

User Actions

Open the link chrome://user-actions, then perform some actions on browser (ex: change tab, reload website, download photos,…) and we have associated events recorded.

Cache

chrome://cache is the entry to complex HTTP cache system.

Suspend request handling in Play Framework 1.x

Simple Future-based class that enables suspending request handling for a given amount of time in an optimal manner.

import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

/**
 * Future-based class, designed to work with play.mvc.Controller.await() for
 * better alternative of Thread.sleep()
 *
 * @author <a href="hoang281283@gmail.com">Minh Hoang TO</a>
 * @date: 10/28/14
 */
public class Wait implements Future<Void> {

    public final long completed_time;

    public Wait(long start_time, long wait_time) {
        completed_time = start_time + Math.min(wait_time, 10000L);
    }

    @Override
    public boolean cancel(boolean mayInterruptIfRunning) {
        return false;
    }

    /**
     * play.mvc.Controller follows the pattern:
     * <p/>
     * if(future.isDone()){
     * try{
     * return future.get();
     * }
     * }
     * <p/>
     * <p/>
     * as illustrated in the Continuation-based implementation
     * <p/>
     * <p/>
     * <p/>
     * if(future.isDone()) {
     * try {
     * return future.get();
     * } catch(Exception e) {
     * throw new UnexpectedException(e);
     * }
     * } else {
     * Request.current().isNew = false;
     * verifyContinuationsEnhancement();
     * storeOrRestoreDataStateForContinuations( false );
     * Continuation.suspend(future);
     * return null;
     * }
     * <p/>
     * That ensures below implementation of isDone() is enough for
     * efficient wait.
     *
     * @return
     */
    @Override
    public boolean isDone() {
        return System.currentTimeMillis() > completed_time;
    }

    @Override
    public Void get() throws InterruptedException, ExecutionException {
        return null;
    }

    @Override
    public Void get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
        return null;
    }

    @Override
    public boolean isCancelled() {
        return false;
    }
}

Usage in Play action method

//Update some data, there is no notification mechanism
//to use Promise in Play and waiting for a hardcoded time
//is the best choice
...
await(new Wait(System.currentTimeMillis(), 5000L);
...
//Read data

Home

Blocking call with CountDownLatch

This post highlights Blocking call with CountDownLatch pattern, a pattern to make synchronous call to asynchronous API thanks to synchronization aid CountDownLatch.

Reasons to use asynchronous API in synchronous manner (which is anti-pattern in most of cases):

• Async requires non-trivial refactoring effort on legacy system
• Async does not effectively increase the throughput (under special contexts), but introduces overall latencies

…….

Asynchronous API

No matter how complex an asynchronous API is, it could be simplified as a service that receives requests from callers and performs callback actions once the responses are available.

public void execute(Request req, ActionListener<Response> callback)

public interface ActionListener<Response>{
   
  void onResponse(Response res);

  void onFailure(Throwable e);
}

Blocking call Pattern

Combination of CountDownLatch with callback that makes caller of execute method suspended until the response is available on caller side.

 final CountDownLatch latch = new CountDownLatch(1);
 execute(req, new ActionListener<Response>{
   public void onResponse(Response res){
     latch.countDown();
   }

   public void onFailure(Throwable e){
     latch.countDown();
   }
 });

 try{
   latch.await();
 }catch(InterruptedException itrEx){

 }

Practical Example

I applied the pattern while integrating ElasticSearch (whose client API is purely asynchronous) into our system.

/**
 * @author <a href="hoang281283@gmail.com">Minh Hoang TO</a>
 * @date: 7/29/14
 */
@Singleton
public class DefaultElasticIndex implements ElasticIndex {
    
    ..........

    @Override
    public void index(AbstractEntity entity, String type, Map<Field, String> entityFields) {
        IndexRequest indexReq = Requests.indexRequest(indexName);
        indexReq.type(type);
        indexReq.id(entity.id);
        indexReq.source(flattern(entity, entityFields));

        if (immediateRefresh) {
            final CountDownLatch indexLatch = new CountDownLatch(1);
            final AtomicBoolean successIndex = new AtomicBoolean(false);
            esClient.index(indexReq, new ActionListener<IndexResponse>() {
                @Override
                public void onResponse(IndexResponse indexResponse) {
                    indexLatch.countDown();
                    successIndex.set(true);
                }

                @Override
                public void onFailure(Throwable e) {
                    indexLatch.countDown();
                }
            });

            try {
                indexLatch.await();
                if (successIndex.get()) {
                    refresh(indexName);
                }
            } catch (InterruptedException itrEx) {
                log.warn("InterruptedException while waiting for synchronous index: " + indexName, itrEx);
            }
        } else {
            esClient.index(indexReq);
        }
    }

    private void refresh(String indexName) {
        RefreshRequest refreshReq = Requests.refreshRequest(indexName);

        final CountDownLatch latch = new CountDownLatch(1);
        esClient.admin().indices().refresh(refreshReq, new ActionListener<RefreshResponse>() {
            @Override
            public void onResponse(RefreshResponse refreshResponse) {
                latch.countDown();
            }

            @Override
            public void onFailure(Throwable e) {
                latch.countDown();
            }
        });

        try {
            latch.await();
        } catch (InterruptedException itrEx) {
            log.warn("InterruptedException while waiting for synchronous refresh of index: " + indexName, itrEx);
        }
    }
    ...............
}   

Home

Discover REST endpoints in ElasticSearch

Learning ElasticSearch REST API often takes newbies a lot of time due to the lack of exhausted list of REST endpoints in ES documentation. Fortunately, ElasticSearch is open-source and its source code could tell us everything.

First, Java classes designed as REST endpoints are under the package org.elasticsearch.rest and its sub-packages.

Package org.elasticsearch.rest

Picking arbitrarily class org.elasticsearch.rest.action.search.RestSearchAction, the class name let us know that the class is about REST endpoints for searching.

Class org.elasticsearch.rest.action.search.RestSearchAction

The content of RestSearchAction tells us:

• URLs and HTTP methods of search REST endpoints.
• Request parameters and their usage.

Home

IntelliJ with JDK 1.7 on OS X

Solution to JDK version issue with IntelliJ on MacBook Pro:

1. Open the file /Applications/IntelliJ\ IDEA\ 13.app/Contents/Info.plist
2. Comment out JVMVersion property as showed in screenshot.

Home