programming

ICFP2010

bwerf — Thu, 24 Jun 2010 10:51:00 GMT

43th place, out of several hundred teams thats not bad. :) (teamname: pluk)

This year the contest was extra intresting for me because I decided to use Pluk instead of C# and in the end i never missed a thing from C#. The compiler needed some fixes and the library some extensions but nothing major.

Really using a language for 72 hours on a difficult challenge is the best way to learn I think.

The Pluk language compiler.

bwerf — Tue, 15 Sep 2009 11:49:00 GMT

http://bitbucket.org/bartwe/plukc/

https://www.ohloh.net/p/plukc

I've moved my compiler online and opensource after working on it for 2 years in private.It currently is in a state that should be good enough to start make it self compiling.

A quick look at the Ohloh statistics show that the project is missing source comments, which is completely true :) Also Ohloh seems to confuse .pluk files with xml or something.

The experience of the previous pluk language project ( http://pluk.sf.net/ ) shows that putting something online doesn't mean anything further will happen. But I hope it can help me build better software and make it easier to discuss language features with other compiler developers.

ICFP2009 contest results.

bwerf — Tue, 15 Sep 2009 11:43:00 GMT

http://icfpcontest.org/

Allways fun to compete in the ICFP contest. This year it went very well and i ended up in 15th place. (teamname: pluk)

Big teams didn't seem to do well, possibly because the problem had few bits that could be developed in parallel.

Can't wait for next year's contest.

Coroutine or continuation?

bwerf — Tue, 20 Jan 2009 21:50:00 GMT

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using System.IO;

using System.Threading;

namespace ConsoleApplication1

{

class Program

{

static void Main(string[] args)

{

var @this = new Program();

Console.WriteLine("Main before");

try

{

@this.Outer().Invoke();

}

catch (Exception e)

{

Console.WriteLine("Main caught exception!");

Console.WriteLine(e);

}

Console.WriteLine("Main after");

Console.ReadLine();

}

IEnumerable<IContinuation> Outer()

{

Console.WriteLine("Outer before");

/* what i want:

* yield Inner2();

yield return Inner2().PrepareYield();

/* What i want to type: (give me a way to extend the c# language and this is possible :)

* try

* {

* int result = yield Inner();

* }

* catch

* {

Console.WriteLine("Caught exception!");

* }

* This does two things, receiving exceptions thrown from a child continuation, and return values.

// What i needed to type:

int result;

Holder<int> r = new Holder<int>();

var y = Inner(r).PrepareYieldAndComplete();

yield return y;

try

{

y.Complete();

result = r.Value;

}

catch (Exception e)

{

Console.WriteLine("Outer caught exception!");

Console.WriteLine(e);

throw;

}

Console.WriteLine("Outer after");

}

IEnumerable<IContinuation> Inner(Holder<int> result)

{

Console.WriteLine("Inner before");

if (result != null)

throw new Exception();

//Console.WriteLine("Inner after");

yield break;

}

IEnumerable<IContinuation> Inner2()

{

Console.WriteLine("Inner2 before");

Console.WriteLine("Inner2 after");

yield break;

}

class Holder<T>

{

public T Value { get; set; }

}

interface IContinuation

{

IEnumerable<WaitHandle> Handles();

bool Process();

}

interface IPassThroughContinuation : IContinuation

{

void Complete();

}

static class ContinationExtensions

{

public static IPassThroughContinuation PrepareYieldAndComplete(this IEnumerable<IContinuation> self)

{

return new PassThroughContinuation(self);

}

public static IContinuation PrepareYield(this IEnumerable<IContinuation> self)

{

return new CombinedContinuation(self);

}

public static bool WaitAny(this IEnumerable<WaitHandle> self)

{

if (self == null)

return true;

foreach (var w in self)

if (w.WaitOne(0, false))

return true;

return false;

}

public static void Invoke(this IEnumerable<IContinuation> self)

{

var c = self.PrepareYield();

while (true)

{

c.Handles().WaitAny();

if (!c.Process())

break;

}

class PassThroughContinuation : IPassThroughContinuation

{

private IEnumerator<IContinuation> source;

private Exception exception;

private bool done;

private IContinuation value;

public PassThroughContinuation(IEnumerable<IContinuation> source)

{

this.source = source.GetEnumerator();

}

public IEnumerable<WaitHandle> Handles()

{

if (done||(value == null))

return null;

return value.Handles();

}

public bool Process()

{

if (done)

return false;

try

{

if (value != null)

if (value.Process())

return true;

if (!source.MoveNext())

{

done = true;

return false;

}

value = source.Current;

return true;

}

catch (ContinuationException ce)

{

exception = ce.InnerException;

done = true;

return false;

}

catch (Exception e)

{

exception = e;

done = true;

return false;

}

public void Complete()

{

if (!done)

throw new InvalidOperationException();

if (exception != null)

throw new ContinuationException(exception);

}

class CombinedContinuation : IContinuation

{

private IEnumerator<IContinuation> source;

private bool done;

private IContinuation value;

public CombinedContinuation(IEnumerable<IContinuation> source)

{

this.source = source.GetEnumerator();

}

public IEnumerable<WaitHandle> Handles()

{

if (done || (value == null))

return null;

return value.Handles();

}

public bool Process()

{

if (done)

return false;

try

{

if (value != null)

if (value.Process())

return true;

if (!source.MoveNext())

{

done = true;

return false;

}

value = source.Current;

return true;

}

catch

{

done = true;

throw;

}

class ContinuationException : Exception

{

public ContinuationException(Exception exception)

: base("Exception raised inside a continuation", exception)

{

}

Yield return continuations

bwerf — Fri, 02 Jan 2009 21:56:00 GMT

Blogging seems to be one of those things i feel i should be doing and with the new year I'm giving it some more attention.

Microsoft TechEd EMEA 2008 in Barcelona was interesting for me. Joe Duffy was there talking about his new concurrency book, a topic that I'm very interested in. A very small but fun session with a large group of core visual studio developers also gave some insights into their thought process.

But most of all a small presentation of CCR DSS, a system made for distributed and concurrent programming that isn't receiving nearly enough attention, mostly due to it having been developed within the robotics team.

One of the things i noticed in the presentation was that they used "yield return" in a way it wasn't designed for. (the best way to use a feature :)

"yield return" can be used to make a normal function turn into a generator:

http://msdn.microsoft.com/en-us/library/9k7k7cf0.aspx

The compiler does a lot of transformations for a piece of code that uses a yield return, transformations that can be used to construct continuations without too much pain.

http://en.wikipedia.org/wiki/Continuation

http://blogs.msdn.com/oldnewthing/archive/2008/08/12/8849519.aspx

When writing server-like programs it is often useful to use continuations to handle a protocol or asynchronous code, where the thread yields the continuation when an operation would have to wait for completion. To do this in c# you can yield return a waitable future of the operation that would block, a top level handler could then decide which continuation is runnable based on this future and execute it. This is more powerful than implementing this using reentrancy because the handler is free to run any continuation it can not just the top most, also the stack wont be exhausted because the continuations use the heap to store method scoped variables.

Example of yield return usage:

void Main() { Handler.Invoke(Save); Handler.Invoke(()=>Process("save")); }

IEnumerable Save() { using (var file = new File()) { yield return file.BeginWrite(); file.EndWrite(); } }

IEnumerable Process(string command) { if (command == "save") foreach (var y in Save()) yield return y; }

Simply calling the Save method would construct an Ienumerable representing the continuation, but not actually invoke Save, this is a serious risk with this pattern, one that a language extension for c# could solve.

The use of the foreach with the yield return in Process is used to pump the waitables, a language extension, for example "yield Save();" would make this more usable.

In this implementation of Process the foreach yield pattern isn't needed, but it helps because it places the execution of Process and Save into the some model, if a simple return was used the Proces method would complete before Save would be invoked, now Save gets executed within the flow of Process in the place it is called.

Exception handling and try/finally is somewhat broken when using yield return, using seems to function, which is surprising since it uses try/finally internally too.

Some issues with the ordering, synchronization and locking start to arise when using this pattern, as with any other system that introduces some form of concurrency.

Clear advantage of this pattern over simply using threads is that it is explicit where the flow is interrupted, so it is easy to keep pre/post-conditions consistent.

The future that the continuation yields can be a task executing on another thread, very handy in places where CPU dependant tasks can be shown to be safely executable on multiple cores with little synchronization.

This made me wonder if the OS switches to another thread when a thread hits a hard page-fault and it needs to do disk IO, probably yes.

Codeview

bwerf — Sat, 26 Jan 2008 15:24:00 GMT

Ugh, took me a day, but i got line numbers working from windbg.

This experience is really making me love open source, at least you have the source code of the receiving end of a system so you can debug where it is going wrong and what it expects.

The problem in this case is the the sstSrcModule documentation is wrong, both the start/end pairs aren't optional but need to have a value, and unlike the spec cannot be filled with zero's. so i now use 0 for start and 0xffffffff for end.

The filenames supposedly used a word for string length but actually uses a byte.

Big thanks to Digital Mars D for implementing a system that apparently ran into similar troubles and figured them out so i also could get past this point.

Also thanks to jaschaW from ddbg.

Links

bwerf — Wed, 23 Jan 2008 12:44:00 GMT

Always handy to have a list of stuff other people find handy in programming languages.

http://repinvariant.blogspot.com/2008/01/thoughts-on-lisp-things-that-other.html

personally i agree with the bignum point, pluk's default int implementation automatically starts to use bignums instead of int when an int overflows, at least that is what i want to implement if i get that far :)

Tail recursion as an optimization might be fun to implement, but what would the call stack look like, also how to detect this at all.

Multi methods is something i don't have a clue about, not sure how to put it into syntax, how to implement it efficiently, what to use it for.

D also has a lot of interest for me, pluk is somewhat similar to it.

http://www.unknownbrackets.com/tutorials/polishing-d

I'm not sure how D will ever become popular without all the fallout ruby is having currently.

This article starts with a slight criticisms that D isn't fully polished yet, but i don't see how an open project ever gets started by being complete and polished.

Looking at the work needed to get a language to the level that seems to be expected seems near impossible :)

Collections

bwerf — Wed, 16 Jan 2008 22:07:00 GMT

After reading about collections in ruby and c#, i'm thinking about how to add collections to pluk.

Collection libraries seem to have become part of what people expect from a language, and seem to basicly consider it a part of it.

Not unreasonably seeing how things like foreach directly talk to its api.

So i'm designing the baseclass that will back most collections, and probably some other 'containers'.

abstract class Iteratable<T>
{
        Iterator<T> createIterator;
        Iteratable<T> copy();
        Iteratable<T> filter(Bool(T) predicate);
        Iteratable<U> map<U>(U(T) converter);
        Iteratable<T> take(Int count);
        Iteratable<T> drop(Int count);
        void foreach(void(T) action);
        U fold<U>(U(U, T) folder, U base);
        Bool exists(Bool(T) predicate);
        Bool all(Bool(T) predicate);
        Bool none(Bool(T) predicate);
}

Things like Sum / Min / Max / Average / Aggregate can be easily be used through the fold method, and so can exists/all/none. Inclusion of directer methods should depend on how commonly something is used, and if it actually saves work/thinking in the long run.

Methods like filter do not return an iteratable with the filtered data but an iteratable that when queried filters based on the filter function from the original iteratable on demand.

This way of lazy processing is more memory efficient generally.

This interface should allow additional syntax sugar like C# LINQ to work with pluk collections.

Programming links

bwerf — Wed, 16 Jan 2008 13:53:00 GMT

http://blog.micropledge.com/2008/01/ten-python-quirkies/

http://blogs.msdn.com/wesdyer/archive/2007/01/03/how-linq-to-objects-queries-work.aspx

windbg plugin example for delphi.

bwerf — Wed, 16 Jan 2008 10:38:00 GMT

It has some issues with shutting down that i'm too lazy to figure out, but it loads and adds a !dumpimage command to windbg that i used to feed a tool i use to find the number of objects in memory used to find memory leaks

library DrBart

// WinDbg plugin that dumps a 4G file named d:\dump\a containing the full memory space of the current open dump/process

// in windbg
// .load DrBart
// .dumpimage

uses
SysUtils,
Classes;

{$R *.res}

type
TExtensionApis = record
    nSize: Cardinal;
    lpOutputRoutine: Pointer;
    lpGetExpressionRoutine: Pointer;
    lpGetSymbolRoutine: Pointer;
    lpDisasmRoutine: Pointer;
    lpCheckControlCRoutine: Pointer;
    lpReadProcessMemoryRoutine: Pointer;
    lpWriteProcessMemoryRoutine: Pointer;
    lpGetThreadContextRoutine: Pointer;
    lpSetThreadContextRoutine: Pointer;
    lpIoctlRoutine: Pointer;
    lpStackTraceRoutine: Pointer;
end;
PExtensionApis = ^ TExtensionApis;
TPrint = procedure(c: PChar); stdcall;
TEXT_API_VERSION = record
    Major, Minor, Rev, Res: Word;
    end;
TReadMem = function (Ptr: Cardinal; Buffer: Pointer; BufLen: Cardinal; Read: Pointer): Cardinal; stdcall;

var
ExtensionApis: TExtensionApis;
ApiVersion: TEXT_API_VERSION;

function dumpimage: integer; Stdcall;
var
I: Integer;
Ptr: Cardinal;
Buf: array[0..511] of Byte;
Read: Cardinal;
fOut: TFileStream;
begin
fout := TFileStream.Create('d:\dump\a', fmCreate);
for I := 0 to $7FFFFF do // 4G / 512
begin
    Ptr := I shl 9;
    TReadMem(ExtensionApis.lpReadProcessMemoryRoutine)(Ptr, @Buf[0], 512, @Read);
    fout.WriteBuffer(Buf[0], 512);
end;
fout.Free;
Result := 0;
end;

procedure WinDbgExtensionDllInit(
    lpExtensionApis: PExtensionApis;
    MajorVersion: Word;
    MinorVersion: Word
    ); stdcall;
begin
    ExtensionApis := lpExtensionApis^;
end;

function
ExtensionApiVersion(): Pointer;
begin
Result := @ApiVersion;
      //
    // ExtensionApiVersion should return EXT_API_VERSION_NUMBER64 in order for APIs
    // to recognize 64 bit addresses. KDEXT_64BIT also has to be defined before including
    // wdbgexts.h to get 64 bit headers for WINDBG_EXTENSION_APIS
    //
    ApiVersion.Rev := 5;
    Result := @ApiVersion;
end;

//
// Routine called by debugger after load
//
procedure
CheckVersion; stdcall;begin end;

exports
WinDbgExtensionDllInit,
ExtensionApiVersion,
CheckVersion,
dumpimage;

begin
end.