struct cli_t
{
  size_t size;                  // Size of this structure
  int32 flags;                  // Feature bits
  const char *sname;            // Short name (displayed on the button)
  const char *lname;            // Long name (displayed in the menu)
  const char *hint;             // Hint for the input line
  // callback: the user pressed Enter
  bool (idaapi *execute_line)(const char *line);

  // callback: the user pressed Tab (optional)
  bool (idaapi *complete_line)(
        qstring *completion,
        const char *prefix,
        int n,
        const char *line,
        int prefix_start);

  // callback: a keyboard key has been pressed (optional)
  bool (idaapi *keydown)(
        qstring *line,
        int *p_x,
        int *p_sellen,
        int *vk_key,
        int shift);
};

static const cli_t cli_python =
{
    sizeof(cli_t),
    0,
    "Python",
    "Python - IDAPython plugin",
    "Enter any Python expression",
    IDAPython_cli_execute_line,
    NULL, // No completion support
    NULL // No keydown support
};

void install_command_interpreter(const cli_t *cp);
void remove_command_interpreter(const cli_t *cp);

The CLI in Python

class pycli_t(idaapi.cli_t):
    flags = 0
    sname = "PyPython"
    lname = "Python - PyCLI"
    hint  = "Enter any Python statement"

    def OnExecuteLine(self, line):
        """
        The user pressed Enter.
        @param line: typed line(s)
        @return Boolean: True-executed line, False-ask for more lines
        """
        try:
            exec(line, globals(), globals())
        except Exception, e:
            print str(e) + "\n" + traceback.format_exc()
        return True

1. Subclass idaapi.cli_t
2. Define the CLI short name, long name and hint
3. Declare the OnExecuteLine handler and use Python's exec() to execute a line

pycli = pycli_t()
pycli.register()

pycli.unregister()

Adding command completion

class cli_t:
    def OnCompleteLine(self, prefix, n, line, prefix_start):
        """
        The user pressed Tab. Find a completion number N for prefix PREFIX

        This callback is optional.

        @param prefix: Line prefix at prefix_start (string)
        @param n: completion number (int)
        @param line: the current line (string)
        @param prefix_start: the index where PREFIX starts in LINE (int)

        @return: None or a String with the completion value
        """
        print "OnCompleteLine: pfx=%s n=%d line=%s pfx_st=%d" % (prefix, n, line, prefix_start)
        return None

• prefix: the prefix at the cursor position
• n: the completion number. If this callback succeeded the first time (when n == 0), then IDA will call this callback again with n=1 (and so on) asking for the next possible completion value
• line: the whole line
• prefix_start: the index of the prefix in the line

OnCompleteLine(prefix="ex", n=0, line="print os.path.ex", prefix_start=14)

1. Parse the identifier: Since IDA passes the line, prefix and the prefix start index, we need to get the whole expression including the prefix (we need "os.path.ex"). This can be done by scanning backwards from prefix_start until we encounter a non identifier character:

   def parse_identifier(line, prefix, prefix_start):
       id_start = prefix_start
       while id_start > 0:
           ch = line[id_start]
           if not ch.isalpha() and ch != '.' and ch != '_':
               id_start += 1
               break
           id_start -= 1
       return line[id_start:prefix_start + len(prefix)]
   

2. Fetch the attributes with dir(): Given the full identifier name (example "os.path.ex"), we split the name by '.' and use a getattr() loop starting from the __main__ module up to the last split value:

   def get_completion(id, prefix):
       try:
           parts = id.split('.')
           m = sys.modules['__main__']
           for i in xrange(0, len(parts)-1):
               m = getattr(m, parts[i])
       except Exception, e:
           return None
       else:
           completion = [x for x in dir(m) if x.startswith(prefix)]
   
           return completion if len(completion) else None
   

3. Implementing OnCompleteLine(): We parse the identifier and get a list of possible completion values only if n==0, otherwise we return the next possible value in the list we previously built:

   def OnCompleteLine(self, prefix, n, line, prefix_start):
       # new search?
       if n == 0:
           self.n = n
           id = self.parse_identifier(line, prefix, prefix_start)
           self.completion = self.get_completion(id, prefix)
       return None if (self.completion is None) or (n >= len(self.completion)) else self.completion[n]
   

f = open('somefile.txt', 'r')
f.re^TAB => f.read, f.readinto, f.readline or f.readlines

print idau^TAB.GetRe^TAB => print idautils.GetRegisterList

idag -Sfirst.idc mydatabase.idb

idag -S"first.idc arg1 arg2" mydatabase.idb

idag -S"first.idc arg1 arg2" -t

#include <idc.idc>

static main()
{
  Message("Hello world from IDC!\n");
  return 0;
}

idag -Sfirst.idc -t

1. Nothing is printed in the console window: This is because the message will show in the output window instead:
   
   
   (It is possible to save all the text in the output window by using the IDALOG environment variable.)
   
   
2. IDA Pro remains open and does not close: To exit IDA Pro when the script finishes, use Exit() IDC function.

Running scripts from the command line

In order to print to the console window, we will not use IDC / Message() instead we will write to a file and when IDA Pro exits we will display the contents of that file.

extern g_idcutil_logfile;
static LogInit()
{
  g_idcutil_logfile = fopen("idaout.txt", "w");
  if (g_idcutil_logfile == 0)
    return 0;
  return 1;
}

static LogWrite(str)
{
  if (g_idcutil_logfile != 0)
    return fprintf(g_idcutil_logfile, "%s", str);
  return -1;
}

static LogTerm()
{
  if (g_idcutil_logfile == 0)
    return;
  fclose(g_idcutil_logfile);
  g_idcutil_logfile = 0;
}

static main()
{
  LogInit(); // Open log file
  LogWrite("Hello world from IDC!\n"); // Write to log file
  LogTerm(); // Close log file

  Exit(0); // Exit IDA Pro
}

idag -Ssecond.idc -t

type idaout.txt

Hello world from IDC!

To simplify this whole process, we wrote a small win32 command line utility called idascript:

IDAScript 1.0 (c) Hex-Rays - A tool to run IDA Pro scripts from the command line

It can be used in two modes:

a) With a database:

        idascript database.idb script.(idc|py|...) [arg1 [arg2 [arg3 [...]]]]

b) With a temporary database:

        idascript script.(idc|py|...) [arg1 [arg2 [arg3 [...]]]]

The script first.idc can now be rewritten like this:

#include <idc.idc>
#include "idascript.idc"

static main()
{
  InitUtils(); // calls LogInit()

  Print(("Hello world from IDC!\n")); // Macro that calls LogWrite()

  Quit(0); // calls LogTerm() following by Exit()
}

import sys

class ToFileStdOut(object):
    def __init__(self):
        self.outfile = open("idaout.txt", "w")

    def write(self, text):
        self.outfile.write(text)

    def flush(self):
        self.outfile.flush()

    def isatty(self):
        return False

    def __del__(self):
        self.outfile.close()

sys.stdout = sys.stderr = ToFileStdOut()

import idc
import idascript

print "Hello world from IDAPython\n"
for i in xrange(1, len(idc.ARGV)):
    print "ARGV[%d]=%s" % (i, idc.ARGV[i])

idc.Exit(0)

Sample scripts

Process list

#include <idc.idc>
#include <idascript.idc>

static main()
{
  InitUtils();

  LoadDebugger("win32", 0);
  auto q = GetProcessQty(), i;

  for (i=0;i<q;i++)
    Print(("[%08X] %s\n", GetProcessPid(i), GetProcessName(i)));

  Quit(0);
}

Kill process

#include <idc.idc>
#include "idascript.idc"
#include "procutil.idc"

static main()
{
  InitUtils();

  // Load the debugger
  LoadDebugger("win32", 0);

  // Get parameters
  if (ARGV.count < 1)
    QuitMsg(0, "Usage: killproc.idc ProcessName\n");

  auto procs = FindProcessByName(ARGV[1]), i;
  if (procs.count == 0)
    QuitMsg(-1, "No process(es) with name " + ARGV[1]);

  for (i=procs.count-1;i>=0;i--)
  {
    auto pid = procs[i];
    Print(("killing pid: %X\n", pid));
    KillProcess(pid);
  }

  Quit(0);
}

D:\idascript>idascript killproc.idc notepad.exe
killing pid: 878
killing pid: 14C8

D:\idascript>

We used here the "ARGV" variable that contains all the parameters passed to IDA Pro via the -S switch, FindProcessByName() utility function and KillProcess() (check procutil.idc)

static KillProcess(pid)
{
  if (!AttachToProcess(pid))
    return 0;

  StopDebugger(); // Terminate the current process

  // Normally, we should get a PROCESS_EXIT event
  GetDebuggerEvent(WFNE_SUSP, -1);
}

Process information

#include "idascript.idc"
#include "procutil.idc"

static DumpProcessInfo()
{
  // Retrieve command line via Appcall
  Print(("Command line: %s\n", GetProcessCommandLine()));

  // Enum modules
  Print(("Module list:\n------------\n"));

  auto x;
  for (x = GetFirstModule();x!=BADADDR;x=GetNextModule(x))
    Print(("Module [%08X] [%08X] %s\n", x, GetModuleSize(x), GetModuleName(x)));

  Print(("\nThread list:\n------------\n"));
  for (x=GetThreadQty()-1;x>=0;x--)
  {
    auto tid = GetThreadId(x);
    Print(("Thread [%x]\n", tid));
    SelectThread(tid);
    Print(("  EIP=%08X ESP=%08X EBP=%08X\n", Eip, Esp, Ebp));
  }
}

static main()
{
  InitUtils();

  // Load the debugger
  LoadDebugger("win32", 0);

  // Get parameters
  if (ARGV.count < 2)
    QuitMsg(0, "Usage: killproc.idc ProcessName\n");

  auto procs = FindProcessByName(ARGV[1]), i;
  for (i=procs.count-1;i>=0;i--)
  {
    auto pid = procs[i];
    if (!AttachToProcess(pid))
    {
      Print(("Could not attach to pid=%x\n", pid));
      continue;
    }
    DumpProcessInfo();
    DetachFromProcess();
  }

  Quit(0);
}

static GetProcessCommandLine()
{
  // Get address of the GetCommandLine API
  auto e, GetCmdLn = LocByName("kernel32_GetCommandLineA");

  if (GetCmdLn == BADADDR)
    return 0;

  // Set its prototype for Appcall
  SetType(GetCmdLn, "char * __stdcall x();");
  try
  {
    // Retrieve the command line using Appcall
    return GetCmdLn();
  }
  catch (e)
  {
    return 0;
  }
}

Extracting function body

#include <idc.idc>
#include "idascript.idc"

static main()
{
  InitUtils();

  if (ARGV.count < 2)
    QuitMsg(0, "Usage: funcextract.idc FuncName OutFile");

  // Resolve name
  auto ea = LocByName(ARGV[1]);
  if (ea == BADADDR)
    QuitMsg(0, sprintf("Function '%s' not found!", ARGV[1]));

  // Get function start
  ea = GetFunctionAttr(ea, FUNCATTR_START);
  if (ea == BADADDR)
    QuitMsg(0, "Could not determine function start!\n");

  // size = end - start
  auto sz = GetFunctionAttr(ea, FUNCATTR_END) - ea;

  auto fp = fopen(ARGV[2], "wb");
  if (fp == 0)
    QuitMsg(-1, "Failed to create output file\n");

  savefile(fp, 0, ea, sz);
  fclose(fp);

  Print(("Successfully extracted %d byte(s) from '%s'", sz, ARGV[1]));
  Quit(0);
}

D:\idascript>idascript ar.idb funcextract.idc start start.bin
Successfully extracted 89 byte(s) from 'start'
D:\idascript>

Other ideas

• Process memory read/write: Check the rwproc.idc script that allows you to read from the process memory to a file or the other way round.
• Associate .IDC with idascript.exe: This allows you to double-click on IDC scripts to run them from the Windows Explorer
• Scriptable debugger: Write scripts to debug a certain process and extract needed information
• ...

Installing the idascript utility

1. Copy idascript.exe to the installation directory of IDA Pro (say %IDA%)
2. Add IDA Pro directory to the PATH environment variable
3. Copy idascript.idc and procutil.idc to %IDA%\idc
4. Copy idascript.py to %IDA%\python
5. Optional: Associate *.idc files with idascript.exe

Debuggers

• added support for MMX/XMM registers:
  
  
  
  
• added more actions to the modules window:
  
  
  
  
  • Load debug symbols: Load additional PDB symbols
  • Jump to module base: Jumps to the module base in the current view
  • Analyze module: Converts the module segments to non-debugger segments and analyzes the module. Handy when analyzing crashdump files
  
  
• added Bochs 2.4.2 support.
  Bochs 2.4.2 introduced range read/write physical watchpoints. If a watchpoint was added from the Bochs command line interface IDA Pro will suspend the execution when the watchpoint triggers.

Bochs Linux debugger plugin

(Debugger selection)

(Debugger configuration)

(Debugger running Under Ubuntu 9 x86)

WinDbg debugger

Apart from bug fixes and minor speed improvements, we added non-invasive debugging support. This ability to attach to processes that are already being debugged comes handy when you want to create crashdumps or inspect handles and other kernel objects.

Make sure you enable this option from the Debugger/Debugger Options/Specific debugger options dialog:

If you are debugging 64-bit applications using idag64, the Windbg plugin will offer to run the debugger server for you automatically:

When the debugger server is no longer needed make sure to terminate it.

Scripting

Processor modules and Plugins

It is now possible to write scriptable loaders, processor modules and plugins. If you always wanted your scripts to automatically execute when a database is loaded and unload/deinitialize when the database is closed, then turn your script into a plugin script with just a few additional lines of code.

If we get enough requests about writing debugger modules using scripts, we may add this facility in the future.

IDAPython improvements

We refactored and improved the IDAPython (now version 1.4.0) plugin (and the extlang_t interface by adding new facilities to call object methods, query properties and so on).
This has lead to significant speed gains as demonstrated by Ero Carrera's blog post.

We also documented all the manually wrapped functions and utility classes which were poorly documented with the example scripts.

The graphical user interface

• The recent scripts window can be configured to be a dockable window or a modal dialog (check idagui.cfg / RECENT_SCRIPTS_MODAL)
• No need to hold the Alt key in order to jump to identifiers, instead simply double click on it
• Output window is now searchable: use Alt-T to start the search and Ctrl-T to search for the next match

Kernel and processor modules

ARM module

We have added support for almost all ARMv7 instructions, including NEON (aka Advanced SIMD). NEON instructions can be found in the code made for Cortex-A8 processors, such as the one in iPhone 3GS and iPad.

idag -parm:ARMv6T2 firmware.bin

ARM_DEFAULT_ARCHITECTURE = "ARMv6";

MIPS module

PC module

Python processor modules

• ebc.py: EFI Byte code processor module:
  
  
  
  
• msp430.py: MSP430 is a simple 27-instructions 16-bit RISC processor from TI.
  
  
  

Closing words

We hope that the new features make your reversing job more easier. Please feel free to send us comments, suggestions and feature requests.

Last but not least, we expect to start the beta testing of the new IDA Qt interface soon. If you are interested and have an active IDA Pro license do not hesitate to contact us.

Every IDC variable is represented by an idc_value_t object in the SDK. The IDC variable type is stored in the idc_value->vtype field and have one of the following values:

• VT_LONG: Used to store 32bit signed numbers (or 64bit numbers if __EA64__ is set). Use idc_value->set_long() to set a value and idc_value->num to read it.
• VT_STR2: Used to store arbitrary strings. Use idc_value->set_string() to set a value, idc_value->c_str() to get a C string or idc_value->qstr() to get a qstring instance.
• VT_INT64: Used to store 64bit signed numbers. Use idc_value->i64 to read the value and idc_value->set_int64() to set a value
• VT_OBJ: Used to represent objects. Such idc variable is created via the VarObject() and attributes are managed by VarSetAttr() / VarGetAttr()
• VT_PVOID: Use to store arbitrary untyped values. Use idc_value->pvoid to read this value and idc_value->set_pvoid() to set it

1. Writing the IDC function callback
2. Registering the function

Writing the callback

typedef error_t idaapi idc_func_t(idc_value_t *argv,idc_value_t *r);

The argv array is initialized by the IDC interpreter and contains the passed arguments and the r argument is set by the callback to return values to the IDC interpreter.

Registering an IDC function

bool set_idc_func(const char *name, idc_func_t *fp, const char *args);

• name: designates the IDC function name to be added
• fp: IDC function callback (written in the previous step)
• args: A zero terminated character array containing the expected arguments types (VT_xxx). For example, an IDC function that expects two numbers and one string as arguments have the following args value: {VT_LONG, VT_LONG, VT_STR2, 0}

Example

static const char idc_getenv_args[] = { VT_STR2, 0 };
static const char idc_getenv_name[] = "getenv";

static error_t idaapi idc_getenv(idc_value_t *argv, idc_value_t *res)
{
  char *env = getenv(argv[0].c_str());
  res->set_string(env == NULL ? "" : env);
  return eOk;
}

int idaapi init()
{
  set_idc_func(idc_getenv_name, idc_getenv, idc_getenv_args );
  return PLUGIN_KEEP;
}

void idaapi term(void)
{
  // Unregister
  set_idc_func(idc_getenv_name, NULL, NULL);
}

Extending IDAPython

• Modifying the source code of IDAPython
• Writing a plugin to extend Python scripting

While the former method requires basic knowledge of SWIG, both methods require some understanding of the Python C API.
In this article, we will use the second method because it is more practical and does not require modification to IDAPython itself.

1. Initializing the Python C API
2. Writing the callback(s)
3. Registering the function(s)

Writing the callback

// ints.hpp
idaman ssize_t ida_export get_predef_insn_cmt(
        const insn_t &cmd,
        char *buf,
        size_t bufsize);

We want to expose it as the following Python function:

def get_predef_insn_cmt(ea):
    """Return instruction comments

    @param ea: Instruction ea
    @return: None on failure or a string containing the instruction comment
    """
    pass
 

static PyObject *py_getpredef_insn_cmt(PyObject * /*self*/, PyObject *args)
{
  do 
  {
    // Parse arguments
    unsigned long ea;
    if (!PyArg_ParseTuple(args, "k", &ea))
      break;

    // Decode instruction
    if (decode_insn(get_screen_ea()) == 0)
      break;

    // Get comments
    char buf[MAXSTR], *p = buf;
    p += qsnprintf(buf, sizeof(buf), "%s: ", ph.instruc[cmd.itype].name);

    if (get_predef_insn_cmt(cmd, p, sizeof(buf) - (p - buf)) == -1 || *p == '\0')
      break;

    // Return comment as a string
    return PyString_FromString(buf);
  } while (false);
  Py_RETURN_NONE;
}

• PyArg_ParseTuple() is used to parse the arguments. This function can be likened to sscanf(). For a description about the format string please refer to the Py_BuildValue() documentation
• get_predef_insn_cmt() is used to fetch the comment and PyString_FromString() is used to return a string Python object
• In case of failures we use the Py_RETURN_NONE macro to return the Py_NONE value (None in Python)

Registering the callback(s)

static PyMethodDef py_methods[] =
{
  {"getpredef_insn_cmt",  py_getpredef_insn_cmt, METH_VARARGS, ""},
  {NULL, NULL, 0, NULL}
};

Py_InitModule("idapyext", py_methods);

Finally, to use this function, make sure you "import idapyext" before calling getpredef_insn_cmt

The source code used in the blog post can be downloaded from here

We received many requests from our clients to make the output window a text control so users select portions of text instead of a whole line.

• Select or delete portions of text
• Jump to any address from the output by Alt-clicking on it

Unified script facilities in the UI

Added "Recent scripts" window

We have dropped the "Recent IDC Scripts" toolbar and replaced it with a script list (similar to IDAPython's Alt-7).
This dialog is common to the text and the graphical interfaces of IDA Pro, and will remember any script (not just IDC). It is accessible via the Windows menu or the Alt-F7 hotkey:

(Note that the history has both Python and IDC scripts)

Added new '-t' command line switch

In the future blog entries we will show how to use this in real life scenarios.

Improved the '-S' command line switch

idag -t -S"hello.idc arg1 arg2 \"arg 3\" arg4"

#include <idc.idc>

static main()
{
  auto i;
  for (i=0;i<ARGV.count;i++)
    Message("ARG[%d]=%s\n", i, ARGV[i]);
}

ARG[0]=hello.idc
ARG[1]=arg1
ARG[2]=arg2
ARG[3]=arg 3
ARG[4]=arg4

import idc

for i in xrange(0, len(idc.ARGV)):
    print "ARG[%d]=%s" % (i, idc.ARGV[i])

Auto-indent in the script input box

If you are interested in participating in the beta testing and you have an active x86 decompiler license, please send us a message. Thanks!

First download the VirtualKd package, unzip its contents and copy the contents of the 'target' folder to the VMWare guest OS and run 'vminstall.exe'. After this step, reboot the guest and run the 'vmmon.exe' (or 'vmmon64.exe') inside the host. If everything is successful, you should see something like this:

Please take note of the 'Pipe name' value as shown in the screenshot. We will use this value when configuring the Windbg plugin.

Configuring IDA Pro / Windbg plugin

First, run IDA Pro without an input database and select Debugger / Attach / Windbg plugin:

Next, we get this screen:

Here we enter a connection string (designating that the com port is a pipe). The pipe name is the same value we read from the 'vmmon' tool from the above steps.

Before pressing OK, we need to make sure that the Windbg plugin is properly configured. Let us verify that by pressing on the 'Debug options' button:

This dialog is used to configure the debugger in general, it is common to all debugger modules. Since we don't need to change any of those settings now, let us instead select 'Set specific options':

Make sure that these two options are properly configured:

1. Kernel debugging is selected
2. Debugging tools path is correctly entered. Please note that even if you're debugging an x64 kernel you still need to point to the x86 version of the debugging tools.

After we configured everything, simply press OK all the way until the attach dialog shows up:

Pressing OK one last time will start the debugging session:

After debugging for a while, we were really amazed at the speed improvement achieved with the help of VirtualKd (nice work VirtualKd team). Kernel debugging speed using the Windbg plugin is almost comparable to the local win32 debugger plugin.

Before being able to write a simple "Hello World" program one needs to know a fair deal about the x86 architecture, the assembler language and the operating system. Obviously this is not the case with high level languages such as C for example.

program helloWorld;
#include( "stdlib.hhf" );
begin helloWorld;
    stdout.put( "Hello, World of Assembly Language", nl );
end helloWorld;

program h;
#include("stdlib.hhf");

static
    s: string := "Hello World!";

procedure chksum_str; @returns( "eax" );
begin chksum_str;
    mov(s, edi);
    xor(ebx, ebx);
    xor(eax, eax);
    while( mov( [edi], al ) <> #0 ) do
        inc(edi); // advance str ptr
        add(eax, ebx);
    endwhile;
    mov(ebx, eax);
end chksum_str;

begin h;
  stdout.put("The checksum of '", s, " is: 0x");
  chksum_str();
  stdout.put(eax);
  if ( eax == 1234 ) then
    stdout.put("Special chksum!", nl);
  endif;
end h;

• Ability to create user types
• Exception handling
• Control and repetition structures (and other constructs found in high level languages)
• Classes and objects
• Various libaries: stl, file i/o, os, array manipulation, math, etc...
• etc...

For example, in Chapter 5 (Procedures and Units), he explains in detail how the stack is set up during a procedure call, how local variables are allocated and how to access the arguments, followed by explanation on how to use HLA to write procedures. Similarly in Chapter 6, when talking about FPU arithmetics, the author carefully explains about the FPU data and controls registers, related instruction set and finally how to use HLA to do FPU arithmetics.

• That always wanted to learn the x86 assembly language but found it difficult to start with. This book is well organized and easy to read
• That want to teach the basics of the x86 architecture and assembly language
• That want to put together an x86 assembly program quickly. HLA compiler and the built-in libraries give you the convenience of a high-level programming language and the power of a low level language. If you use the standard libraries provided by HLA then your program is portable

• GetEnvironmentStrings: Returns a block of memory pointing to all the environment variables in the process. This block is a list of zero terminated strings, the end of the list is marked with two \0 characters (such list is also known as MULTI_SZ)
• FreeEnvironmentStrings: Frees the block returned by GetEnvironmentStrings()
• SetEnvironmentVariable: Creates or edits an environment variable

Setting up Appcall

GetEnvironmentStrings = 
    Appcall.proto("kernel32_GetEnvironmentStringsA", 
                  "unsigned int __stdcall GetEnvironmentStrings();")

SetEnvironmentVariable = 
    Appcall.proto("kernel32_SetEnvironmentVariableA", 
                  "int __stdcall SetEnvironmentVariable(const char *lpName, const char *lpValue);")

FreeEnvironmentStrings = 
    Appcall.proto("kernel32_FreeEnvironmentStringsA", 
                  "int __stdcall FreeEnvironmentStrings(unsigned int block);")

# Add a new environment variable
SetEnvironmentVariable("MY_VAR", "Some value")

• Both GetEnvironmentStrings() and FreeEnvironmentStrings() prototypes are re-declared to accept an unsigned int instead of a character pointer because Appcall does not support the MULTI_SZ type.
• We are using the ASCII version of those APIs. Modifying the script to work with the unicode version is not very difficult to achieve.

Retrieving all environment variables

1. Call the GetEnvironmentStrings() to retrieve a pointer to the environment block in the process. Save that pointer
2. Read the debuggee's memory at that block and retrieve the variables accordingly.
3. Free the block by calling FreeEnvironmentStrings()

def Variables():
    """Returns all environment blocks"""

    # Get all environment strings
    env_ptr = GetEnvironmentStrings()
    if env_ptr == 0:
        return None

    # Always refresh the memory after a call 
    # that could change the memory configuration
    idc.RefreshDebuggerMemory()

    # Open process memory
    f = idaapi.loader_input_t()
    f.open_memory(env_ptr)

    # Parse all environment variables into a list of variables
    vars = []
    var = []
    while True:
        ch = f.get_char()
        if not ch:
            break
        if ch == '\x00':
            # double-null -> end of list
            if not var:
                break
            vars.append(''.join(var))
            var = []
        else:
            var.append(ch)
    f.close()

    # Free the block
    FreeEnvironmentStrings(env_ptr)

    return vars

Writing a custom viewer to manage the environment variables

• Retrieves all the environment variables: call GetEnvironmentStrings() and parse the result
  • Parse each environment variable into KEY/VALUE components
  • Add a colored line into the custom viewer with different colors for the KEY and VALUE
• Add popup menu entries and handle keypresses to support three operations:
  • Insert: Ask the user for a key and value using the idaapi.askstr() then call the SetEnvironmentVariable() API as shown in the previous example
  • Edit: Retrieve the cursor position, extract the key name, ask the user for a new value only (w/o asking for a key) and call the SetEnvironmentVariable() API
  • Delete: Retrieve the key name and call SetEnvironmentVariable() API with value pointing to NULL

Please download the script from here.

(A plugin script written using IDC)

• init/run/term callbacks: These callbacks are called when IDA initializes, invokes and terminates the plugin.
• flags: The plugin flags describe the kind of the plugin. A plugin could have no flags (flags = 0) which means that this is an ordinary plugin. Or the plugin could act as a processor module extension, thus the PLUGIN_PROC flag. Another type of plugins are the debugger plugins which use the PLUGIN_DBG flag.
• Description and hotkey: The remaining plugin_t entries are used to designate a name, help text, comment, and a hotkey for the plugin.

• PLUGIN_OK: Plugin agrees to work with the current database but will be loaded only when it is about to be invoked.
• PLUGIN_SKIP: Plugin does not want to work with the current database. For example a plugin may choose to work with certain input file types.
• PLUGIN_KEEP: Plugin agrees to work with the current database and will remain loaded until the database is closed. It is important to return this value if you hook to notification events or register any sort of callbacks.

int idaapi run(int arg)

; plugin_name     filename     hotkey  arg  [flags]
Extract_File      extract      0       1
Extract_Block     extract      0       2
Extract_Item      extract      Alt-F8  3

If a plugin is not present in plugins.cfg and is invoked using the "Edit/Plugins" menu or with the hotkey described in its PLUGIN entry then run() will be invoked with arg=0. Configured plugins, when invoked, will pass to the run() the argument value specified in the configuration file.

Scriptable plugins

import idaapi

class myplugin_t(idaapi.plugin_t):
    flags = idaapi.PLUGIN_UNL
    comment = "This is a comment"
    help = "This is help"
    wanted_name = "My Python plugin"
    wanted_hotkey = "Alt-F8"

    def init(self):
        idaapi.msg("init() called!\n")
        return idaapi.PLUGIN_OK

    def run(self, arg):
        idaapi.msg("run() called with %d!\n" % arg)

    def term(self):
        idaapi.msg("term() called!\n")

def PLUGIN_ENTRY():
    return myplugin_t()

Scriptable plugins are so easy to write, and deploy (a simply copy/paste in the plugins folder) and very useful in case you want your plugin to load and unload automatically like native plugins.

v = idaapi.simplecustviewer_t()
if v.Create("Simple custom viewer"):
    for i in xrange(1, 11): 
        v.AddLine("Line %d" % i)
    v.Show()
else:
    print "Failed to create viewer"

class mycv_t(simplecustviewer_t):
    def Create(self, sn=None):
        # Form the title
        title = "Simple custom view test"
        if sn:
            title += " %d" % sn

        # Create the customview
        if not simplecustviewer_t.Create(self, title):
            return False

        self.menu_hello = self.AddPopupMenu("Hello")
        self.menu_world = self.AddPopupMenu("World")

        for i in xrange(0, 100):
            self.AddLine("Line %d" % i)

        return True

    def OnKeydown(self, vkey, shift):
        # ESCAPE?
        if vkey == 27:
            self.Close()
        # Goto?
        elif vkey == ord('G'):
            n = self.GetLineNo()
            if n is not None:
                v = idc.AskLong(self.GetLineNo(), "Where to go?")
                if v:
                    self.Jump(v, 0, 5)
        elif vkey == ord('R'):
            print "refreshing...."
            self.Refresh()
        else:
            return False
        return True

    def OnPopupMenu(self, menu_id):
        if menu_id == self.menu_hello:
            print "Hello"
        elif menu_id == self.menu_world:
            print "World"
        else:
            # Unhandled
            return False
        return True

view = mycv_t()
if view.Create(1):
    view.Show()

def make_many(n):
    L = []
    for i in xrange(1, n+1):
        v = mycv_t()
        if not v.Create(i):
            break
        v.Show()
        L.append(v)
    return L
# Create 20 views
V = make_many(20)

f = idaapi.find_tform("Simple custom view test 2")
if f:
    idaapi.close_tform(f, 0)

For a more comprehensive example on custom viewers, please check the ex_custview.py example.

Using colored lines

To use colored lines, we have to embed color tags to them. All the available foreground colors are defined in lines.hpp header file. The color codes are related to various item kinds in IDA, for example here are some colors:

There are also special color tags treated as escape sequence codes (the concept is similar to ANSI escape codes). They are used to determine how a line is rendered, to mark the beginning/end of a certain color, to insert address marks, or to mark UTF8 string beginnings/endings:

//      A typical color sequence looks like this:
//
//      COLOR_ON COLOR_xxx text COLOR_OFF COLOR_xxx

colored_line = idaapi.COLSTR("Hello", idaapi.SCOLOR_REG) + " " + idaapi.COLSTR("World", idaapi.SCOLOR_STRING)

'\x01!Hello\x02! \x01\x0bWorld\x02\x0b'

COLOR_ON SCOLOR_REG=0x21 Hello COLOR_OFF COLOR=0x21 SPACE COLOR_ON SCOLOR_STRING=0x0B World COLOR_OFF COLOR=0x0B

In order to strip back color tags from a colored line, use tag_remove():

line = idaapi.tag_remove(colored_line)

Writing an ASM file viewer

1. ASM tokenizer: It should be able to recognize comments, strings and identifiers. For the identifiers, we will take into consideration only register names, instruction names and directives.
   • Instruction names: To get all the instruction names we use idautils.GetInstructionList() which returns all the instruction names from the processor module (the ph.instruc array)
   • Register names: Similarly we can use idautils.GetRegisterList()
2. Custom viewer to render the text: We derive from simplecustviewer_t to handle key presses and popup menu actions

The tokenizer (asm_colorizer_t class) will go over the text and when it identifies a token it will call one of the following functions: as_string(), as_comment(), as_num(), and as_id(). Those functions will use idaapi.COLSTR() to colorize the token appropriately. At the end of each line, the tokenizer will call the add_line() method to add the line (after it has been colored).

class asmview_t(idaapi.simplecustview_t, asm_colorizer_t):
    def Create(self, fn):
        # Create the customview
        if not idaapi.simplecustview_t.Create(self, "ASM View - %s" % os.path.basename(fn)):
            return False

        self.instruction_list = idautils.GetInstructionList()
        self.instruction_list.extend(["ret"])
        self.register_list    = idautils.GetRegisterList()
        self.register_list.extend(["eax", "ebx", "ecx", "edx", "edi", "esi", "ebp", "esp"])

        self.fn = fn
        if not self.reload_file():
            return False

        self.id_refresh = self.AddPopupMenu("Refresh")
        self.id_close   = self.AddPopupMenu("Close")

        return True

    def reload_file(self):
        if not self.colorize_file(self.fn):
            self.Close()
            return False
        return True

    def colorize_file(self, fn):
        try:
            f = open(fn, "r")
            lines = f.readlines()
            f.close()
            self.ClearLines()
            self.colorize(lines)
            return True
        except:
            return False

    def add_line(self, s=None):
        if not s:
            s = ""
        self.AddLine(s)

    def as_comment(self, s):
        return idaapi.COLSTR(s, idaapi.SCOLOR_RPTCMT)

    def as_id(self, s):
        t = s.lower()
        if t in self.register_list:
            return idaapi.COLSTR(s, idaapi.SCOLOR_REG)
        elif t in self.instruction_list:
            return idaapi.COLSTR(s, idaapi.SCOLOR_INSN)
        else:
            return s

    def as_string(self, s):
        return idaapi.COLSTR(s, idaapi.SCOLOR_STRING)

    def as_num(self, s):
        return idaapi.COLSTR(s, idaapi.SCOLOR_NUMBER)

    def as_directive(self, s):
        return idaapi.COLSTR(s, idaapi.SCOLOR_KEYWORD)

    def OnPopupMenu(self, menu_id):
        if self.id_refresh == menu_id:
            return self.reload_file()
        elif self.id_close == menu_id:
            self.Close()
            return True
        return False

    def OnKeydown(self, vkey, shift):
        # ESCAPE
        if vkey == 27:
            self.Close()
            return True
        return False

This blog entry inspired you to write a new plugin? Feel free to participate in our plugin contest!

Qt (pronounced "cute") is a cross-platform application and UI framework and the Win32 VCL-based IDA Pro interface is being ported to it. The goal is to provide all the features available in the current GUI while maintaining the maximum compatibility with plugins and other external modules.

Here is a screenshot of the current build of idaqt running on Ubuntu:

You can click on the images to enlarge them.

This is idaqt on Windows 7. The text view looks exactly the same and all other features like choosers and forms will be available with no exception on all supported platforms.

The full range of options and customizations which the Win32 interface provides will be available as well.

As you can see, apart from the still to be implemented docking, the new interface looks pretty much the same as the Win32 one.

Not only will it be possible to deploy the same native graphical interface to Windows, OS X, Linux and other platforms which in the future may become popular, but the quality of the user experience and the further development capabilities will be hugely increased thanks to an advanced framework such as Qt.

Although idaqt is going to replace the current GUI completely, for some time they will be deployed together in order to fix any incompatibility issues and to give third party developers the necessary time to thoroughly test their products against the new interface.

What are custom types and formats

• Custom data type: A custom type is basically just a way to tag some bytes for later display with custom format, when the built-in IDA types (dt_byte, dt_word, etc) are not enough. For example: an XMM vector, a Pascal string, a half-precision (16 bits) floating-point number, a 16:32 far pointer (fword), uleb128 number and so on. To define a custom type, you need to provide its name, size (fixed or dynamically calculated), keyword for disassembly and a few other attributes.
• Custom data format: The custom data format allows you do display a custom or built-in data type in any way you like. You can register several formats for each type and switch the representation. For example, you might want to switch the display of the same 16-byte XMM vector between four floats or two doubles. A format definition includes callback for printing (to display) and scanning (used during debugging to change the register values).

Its implementation is very simple:

Next we illustrate some possible usages of custom types and formats. Other uses are also possible too, it is up to your imagination.

Decoding embedded bytecodes

• Write a dedicated processor module and then load the extracted bytecodes separately
• Or define the bytecodes as bytes and then use comments to describe the real meaning of those bytecodes.

With this new addition, one can just write a custom data type to handle the situation:

And if you happen to have a situation where the bytecodes are operands to instructions (as means of obfuscation), you can still apply the custom format on those operands:

The previous blog entry showed how to write processor modules using Python. What if one simply uses the "import" statement to import a full-blown processor module script and use it in the custom data types/formats? ;)

Displaying resource strings

When reversing MS Windows applications, one can encounter string IDs, but then how to easily and nicely go fetch the data and display it in the disassembly listing?
Normally, one would have to use a resource editor to extract the string value corresponding to the string id, then to create an enum in IDA for each string ID with a repeatable comment:

That works, but what about writing your own custom format instead:

And then applying it directly without having to use a resource editor to extract the string value, have the custom format do that programmatically for you :

This is how a resource string custom format handler can look like:

To take a closer look at it, you can download the custom data type handler script along with the source code of the simplevm assembler/disassembler and the C program that was used in this article.
]]> tag:hexblog.com,2010://1.113 2010-02-16T17:38:51Z 2010-03-22T11:21:33Z

One of the new features we are preparing for the next version of IDA is the ability to write processor modules using your favorite scripting language.
After realizing how handy it is to write file loaders using scripting languages, we set out to making the same thing for processor modules. As an exercise for this new feature, we implemented a processor module for the EFI bytecode.

]]> Background In IDA Pro, a processor module implementation is usually split into four parts:

1. Processor, assembler, instructions and registers definitions (ins.cpp/.hpp, reg.cpp)
2. Decoder (ana.cpp): decodes an instruction into an insn_t structure (the 'cmd' global variable)
3. Emulation (emu.cpp): emulates instructions, creates appropriate cross references, traces the stack, recognizes code patterns, etc...
4. Output (out.cpp): outputs the result to the screen

The processor module is described using the processor_t structure. It holds pointers to registers, instructions, processor module name and other callbacks (ana, emu, out, notify, ...).
The assembler is described using the asm_t structure. It holds pointers to the assembler syntax and other callbacks.
For more information about structures and functions used in IDA API and processor modules (e.g. insn_t), see this great tutorial by Steve Micallef.

Writing a processor module in Python

To write a processor module in Python, we follow similar logic.

1. Write the get_idp_desc() function. It simply tells IDA what processors the module can handle.

   def get_idp_desc():
       return "EFI Byte code:ebc"
   

   The return value means that this processor is named "EFI Byte code" and its shortname is "ebc". Thus a subsequent call to set_processor_type('ebc') from the part of a file loader will succeed.

   In case of the pc processor module, which can handle many variations of x86 architecture, the string looks like this:

   Intel 80x86 processors:8086:80286r:80286p:80386r:80386p:...

2. Define the registers and instructions:

   # Registers definition
     proc_Registers = [
         # General purpose registers
         "R0",
         "R1",
         ...,
         "R10",
         ...
     ]
   
     # Instructions definition
     proc_Instructions = [
         {'name': 'INSN1', 'feature': CF_USE1},
         {'name': 'INSN2', 'feature': CF_USE1 | CF_CHG1}
         ...
     ]
     

3. Write the get_idp_def() function. It should return a dictionary similar to the processor_t structure with the processor, assembler, instructions and registers definitions.
   

   # This function returns the processor module definition
   def get_idp_def():
       return {
           'version': IDP_INTERFACE_VERSION,
   
           # IDP id
           'id' : 0x8000 + 1,
   
           # Processor features
           'flag' : PR_USE32 | PRN_HEX | PR_RNAMESOK,
   
           # short processor names
           # Each name should be shorter than 9 characters
           'psnames': ['ebc'],
   
           # long processor names
           # No restriction on name lengthes.
           'plnames': ['EFI Byte code'],
   
           # number of registers
           'regsNum': len(proc_Registers),
   
           # register names
           'regNames': proc_Registers,
         
           # Array of instructions
           'instruc': proc_Instructions,
           ....
           'assembler': \
           {
                   # flag
                   'flag' : ASH_HEXF3 | AS_UNEQU | AS_COLON | ASB_BINF4 | AS_N2CHR,
   
                   # Assembler name (displayed in menus)
                   'name': "EFI bytecode assembler",
                   ...
                   # byte directive
                   'a_byte': "db",
   
                   # word directive
                   'a_word': "dw",
   
                   # remove if not allowed
                   'a_dword': "dd",
                   ...
   
           } # Assembler
       }
   

Now that we finished all the declarations, we can implement the decoder (or analyzer), emulator and the output callbacks.

• The analyzer looks like this:

  def ph_ana():
      """
      Decodes an instruction into the global variable 'cmd'
      Current address is pre-filled in cmd.ea
      """
      cmd = idaapi.cmd
  
      # take opcode byte
      b = ua_next_byte()
      # decode and fill cmd.Operands etc...
      # ...
  
      # Return decoded instruction size or zero
      return cmd.size
  

  
  And decoding one instruction/filling the 'cmd' variable may look like this:

  def decode_JMP8(opbyte, cmd):
      conditional   = (opbyte & 0x80) != 0
      cs            = (opbyte & 0x40) != 0
      cmd.Op1.type  = o_near
      cmd.Op1.dtyp  = dt_byte
      addr          = ua_next_byte()
      cmd.Op1.addr  = (as_signed(addr, 8) * 2) + cmd.size + cmd.ea
  
      if conditional:
          cmd.auxpref = FL_CS if cs else FL_NCS
  
      return True
  

• The emulator:

  # Emulate instruction, create cross-references, plan to analyze
  # subsequent instructions, modify flags etc. Upon entrance to this function
  # all information about the instruction is in 'cmd' structure.
  # If zero is returned, the kernel will delete the instruction.
  def ph_emu():
      aux = cmd.auxpref
      Feature = cmd.get_canon_feature()
  
      if Feature & CF_USE1:
          handle_operand(cmd.Op1, 1)
      if Feature & CF_CHG1:
          handle_operand(cmd.Op1, 0)
      if Feature & CF_USE2:
          handle_operand(cmd.Op2, 1)
      if Feature & CF_CHG2:
          handle_operand(cmd.Op2, 0)
      if Feature & CF_JUMP:
          QueueMark(Q_jumps, cmd.ea)
  
      # add flow xref
      if Feature & CF_STOP == 0:
          ua_add_cref(0, cmd.ea + cmd.size, fl_F)
  
      return 1
  

• The output callback:

  # Generate text representation of an instruction in 'cmd' structure.
  # This function shouldn't change the database, flags or anything else.
  # All these actions should be performed only by ph_emu() function.
  def ph_out():
      cmd = idaapi.cmd
      # Init output buffer
      buf = idaapi.init_output_buffer(1024)
  
      # First, output the instruction mnemonic
      OutMnem()
  
      # Output the first operand if present (this invokes the ph_outop callback)
      out_one_operand( 0 )
  
      # Output the rest of the operands
      for i in xrange(1, 3):
          op = cmd[i]
  
          if op.type == o_void:
              break
  
          out_symbol(',')
          OutChar(' ')
          out_one_operand(i)
  
      # Terminate the output buffer
      term_output_buffer()
  
      # Emit the line
      cvar.gl_comm = 1
      MakeLine(buf)
  

  Note that the previous callbacks are very similar to their C language counterparts.

Although this feature will not work with the current version of IDA Pro, you can download the EBC script sample for a preview of how a module would look.

If you like this feature, make sure to apply for the beta testing of next version when we announce it!

]]>