Python -- Scripting language which supports coroutines

**May the source be with you, but remember the KISS principle ;-)** · activating

def __init__(self, **keyword_args):

self.theme_name = "default"

#!/usr/bin/env python

import screenlets

class HelloWorldScreenlet(screenlets.Screenlet):
    __name__ = 'HelloWorld'
    __version__ = '0.1'
    __author__ = 'John Doe'
    __desc__ = 'Simple Hello World Screenlet'

    def __init__(self, **kwargs):
        # Customize the width and height.
        screenlets.Screenlet.__init__(self, width=180, height=50, **kwargs)

    def on_draw(self, ctx):
        # Change the color to white and fill the screenlet.
        ctx.set_source_rgb(255, 255, 255)
        self.draw_rectangle(ctx, 0, 0, self.width, self.height)

        # Change the color to black and write the message.
        ctx.set_source_rgb(0, 0, 0)
        text = 'Hello World!'
        self.draw_text(ctx, text, 10, 10, "Sans 9" , 20, self.width)

if __name__ == "__main__":
    import screenlets.session
    screenlets.session.create_session(HelloWorldScreenlet)

#!/usr/bin/env python

import screenlets
import cairo
import datetime

class TimeTrackerScreenlet(screenlets.Screenlet):
	__name__ = 'TimeTrackerScreenlet'
	__version__ = '0.1'
	__author__ = 'John Doe'
	__desc__ = 'A basic time tracker screenlet.'

	theme_dir = 'themes/default'
	image = 'start.png'

	def __init__(self, **keyword_args):
		screenlets.Screenlet.__init__(self, width=250, height=50, **keyword_args)
		self.add_default_menuitems()
		self.y = 25
		self.theme_name = 'default'
		self.on = False
		self.started = None

	def on_draw(self, ctx):
		self.draw_scaled_image(ctx, 0, 0, self.theme_dir + '/' + 
		self.image, self.width, self.height)

	def on_mouse_down(self, event):
		if self.on:
			self.started = datetime.datetime.now()
			self.image = 'stop.png'
			self.on = False
		else:
			if self.started:
				length = datetime.datetime.now() - self.started
				screenlets.show_message(None, '%s seconds' % 
				length.seconds, 'Time')
				self.started = None
			self.image = 'start.png'
			self.on = True

	def on_draw_shape(self, ctx):
		self.on_draw(ctx)
		ctx.rectangle(0, 0, self.width, self.height)
		ctx.fill()

if __name__ == "__main__":
	import screenlets.session
	screenlets.session.create_session(TimeTrackerScreenlet)

#!/usr/bin/env python

from screenlets.options import StringOption, IntOption, ListOption
import xml.dom.minidom
import webbrowser
import screenlets
import urllib2
import gobject
import pango
import cairo

class RSSSearchScreenlet(screenlets.Screenlet):
    __name__ = 'RSSSearch'
    __version__ = '0.1'
    __author__ = 'John Doe'
    __desc__ = 'An RSS search screenlet.'

    topic = 'Windows Phone 7'
    feeds = ['http://www.engadget.com/rss.xml',
             'http://feeds.gawker.com/gizmodo/full']
    interval = 10

    __items = []
    __mousesel = 0
    __selected = None

    def __init__(self, **kwargs):
        # Customize the width and height.
        screenlets.Screenlet.__init__(self, width=250, height=300, **kwargs)
        self.y = 25

    def on_init(self):
        # Add options.
        self.add_options_group('Search Options',
                               'RSS feeds to search and topic to search for.')
        self.add_option(StringOption('Search Options',
            'topic',
            self.topic,
            'Topic',
            'Topic to search feeds for.'))
        self.add_option(ListOption('Search Options',
                                   'feeds',
                                   self.feeds,
                                   'RSS Feeds',
                                   'A list of feeds to search for a topic.'))
        self.add_option(IntOption('Search Options',
                                  'interval',
                                  self.interval,
                                  'Update Interval',
                                  'How frequently to update (in seconds)'))

        self.update()

    def update(self):
        """Search selected feeds and update results."""

        self.__items = []

        # Go through each feed.
        for feed_url in self.feeds:

            # Load the raw feed and find all item elements.
            raw = urllib2.urlopen(feed_url).read()
            dom = xml.dom.minidom.parseString(raw)
            items = dom.getElementsByTagName('item')

            for item in items:

                # Find the title and make sure it matches the topic.
                title = item.getElementsByTagName('title')[0].firstChild.data
                if self.topic.lower() not in title.lower(): continue

                # Shorten the title to 30 characters.
                if len(title) > 30: title = title[:27]+'...'

                # Find the link and save the item.
                link = item.getElementsByTagName('link')[0].firstChild.data
                self.__items.append((title, link))

        self.redraw_canvas()

        # Set to update again after self.interval.
        self.__timeout = gobject.timeout_add(self.interval * 1000, self.update)

    def on_draw(self, ctx):
        """Called every time the screenlet is drawn to the screen."""

        # Draw the background (a gradient).
        gradient = cairo.LinearGradient(0, self.height * 2, 0, 0)
        gradient.add_color_stop_rgba(1, 1, 1, 1, 1)
        gradient.add_color_stop_rgba(0.7, 1, 1, 1, 0.75)
        ctx.set_source(gradient)
        self.draw_rectangle_advanced (ctx, 0, 0, self.width - 20,
                                      self.height - 20,
                                      rounded_angles=(5, 5, 5, 5),
                                      fill=True, border_size=1,
                                      border_color=(0, 0, 0, 0.25),
                                      shadow_size=10,
                                      shadow_color=(0, 0, 0, 0.25))

        # Make sure we have a pango layout initialized and updated.
        if self.p_layout == None :
            self.p_layout = ctx.create_layout()
        else:
            ctx.update_layout(self.p_layout)

        # Configure fonts.
        p_fdesc = pango.FontDescription()
        p_fdesc.set_family("Free Sans")
        p_fdesc.set_size(10 * pango.SCALE)
        self.p_layout.set_font_description(p_fdesc)

        # Display our text.
        pos = [20, 20]
        ctx.set_source_rgb(0, 0, 0)
        x = 0
        self.__selected = None
        for item in self.__items:
            ctx.save()
            ctx.translate(*pos)

            # Find if the current item is under the mouse.
            if self.__mousesel == x and self.mouse_is_over:
                ctx.set_source_rgb(0, 0, 0.5)
                self.__selected = item[1]
            else:
                ctx.set_source_rgb(0, 0, 0)

            self.p_layout.set_markup('%s' % item[0])
            ctx.show_layout(self.p_layout)
            pos[1] += 20
            ctx.restore()
            x += 1

    def on_draw_shape(self, ctx):
        ctx.rectangle(0, 0, self.width, self.height)
        ctx.fill()

    def on_mouse_move(self, event):
        """Called whenever the mouse moves over the screenlet."""

        x = event.x / self.scale
        y = event.y / self.scale
        self.__mousesel = int((y -10 )/ (20)) -1
        self.redraw_canvas()

    def on_mouse_down(self, event):
        """Called when the mouse is clicked."""

        if self.__selected and self.mouse_is_over:
            webbrowser.open_new(self.__selected)

if __name__ == "__main__":
    import screenlets.session
    screenlets.session.create_session(RSSSearchScreenlet)

raw = urllib2.urlopen(feed_url).read()
dom = xml.dom.minidom.parseString(raw)
items = dom.getElementsByTagName('item')

for item in items:
    # Find the title and make sure it matches the topic.
    title = item.getElementsByTagName('title')[0].firstChild.data
    if self.topic.lower() not in title.lower(): continue

>>> r = range(10)

>>> i = iter(r)

>>> x = xrange(10)

>>> g = itertools.takewhile(lambda n: n<10, itertools.count())

#...etc...

>>> r[4]
4

>>> i[4]
TypeError: unindexable object

>>> thing, temp = itertools.tee(thing)

>>> zip(temp, '.'*5)[-1][0]
4

>>> r[4:9:2]
[4, 6, 8]

>>> list(itertools.islice(r,4,9,2))  # works for iterators
[4, 6, 8]

>>> class Indexable(object):
...     def __init__(self, it):
...         self.it = it
...     def __getitem__(self, x):
...         self.it, temp = itertools.tee(self.it)
...         if type(x) is slice:
...             return list(itertools.islice(self.it, x.start, x.stop, x.step))
...         else:
...             return zip(temp, range(x+1))[-1][0]
...     def __iter__(self):
...         self.it, temp = itertools.tee(self.it)
...         return temp
...

>>> integers = Indexable(itertools.count())

>>> integers[4]
4
>>> integers[4:9:2]
[4, 6, 8]

with
with open('/etc/passwd', 'r') as f:
    for line in f:
        print line
        ... more processing code ...
cout << ( a==b ? "first option" : "second option" )

dir(5)

['__abs__', '__add__', '__and__', '__class__', '__cmp__', '__coerce__', '__delattr__', '__div__', 
'__divmod__', '__doc__', '__float__', '__floordiv__', '__getattribute__', '__getnewargs__', 
'__hash__', '__hex__', '__init__', '__int__', '__invert__', '__long__', '__lshift__', '__mod__',
'__mul__', '__neg__', '__new__', '__nonzero__', '__oct__', '__or__', '__pos__', '__pow__', '
__radd__', '__rand__', '__rdiv__', '__rdivmod__', '__reduce__', '__reduce_ex__', '__repr__',
'__rfloordiv__', '__rlshift__', '__rmod__', '__rmul__', '__ror__', '__rpow__', '__rrshift__',
'__rshift__', '__rsub__', '__rtruediv__', '__rxor__', '__setattr__', '__str__', '__sub__', '__truediv__', '__xor__']

def foo()
      pass
...
dir(foo)

['__call__', '__class__', '__delattr__', '__dict__', '__doc__', '__get__', '__getattribute__',
'__hash__', '__init__', '__module__', '__name__', '__new__', '__reduce__', '__reduce_ex__',
'__repr__', '__setattr__', '__str__', 'func_closure', 'func_code', 'func_defaults', 'func_dict', 
'func_doc', 'func_globals', 'func_name']

>>> class A(object):
...     def __init__(self):
...             self.x = 3
...     def dump(self):
...             print self.x
...
>>> dir(A)

['__class__', '__delattr__', '__dict__', '__doc__', '__getattribute__', '__hash__', '__init__', 
'__module__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__str__', 
'__weakref__', 'dump']

>>> a = A()
>>> dir(a)

['__class__', '__delattr__', '__dict__', '__doc__', '__getattribute__', '__hash__', '__init__',
'__module__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__str__',
'__weakref__', 'dump', 'x']

>>> import types
>>> dir(types)

['BooleanType', 'BufferType', 'BuiltinFunctionType', 'BuiltinMethodType', 'ClassType', 'CodeType',
'ComplexType', 'DictProxyType', 'DictType', 'DictionaryType', 'EllipsisType', 'FileType', 
'FloatType', 'FrameType', 'FunctionType', 'GeneratorType', 'InstanceType', 'IntType', 'LambdaType',
'ListType', 'LongType', 'MethodType', 'ModuleType', 'NoneType', 'NotImplementedType', 'ObjectType',
'SliceType', 'StringType', 'StringTypes', 'TracebackType', 'TupleType', 'TypeType', 
'UnboundMethodType', 'UnicodeType', 'XRangeType', '__builtins__', '__doc__', '__file__', '__name__']

>>> type(None)
<type 'NoneType'>

>>> type(5)
<type 'int'>

>>> x = [1,2,3]
>>> type(x)
<type 'list'>

>>> type(list)
<type 'type'> 

>>> type(type)
<type 'type'>

>>> type(object)
<type 'type'>
>>>

>>> import types
>>> type(types)
<type 'module'>

>>> type==types.TypeType
True

>>> class A(object):
...     pass

>>> a = A()

>>> type(A)
<type 'type'>

>>> type(a)
<class '__main__.A'>

>>> a.__class__
<class '__main__.A'>

>>> type(object)
<type 'type'>

>>> type.__bases__
(<type 'object'>,)

>>> object.__bases__
()

def init_method(self, x, y):
      self.x = x
      self.y = y
def dumpSum_method(self):
      print self.x + self.y

D = type('DynamicClass',
                   (object,),
                   {'__init__':init_method, 'dumpSum':dumpSum_method})
d = D(3, 4)
d.dumpSum()

class A(object):
    def foo(self):
        print 'I am foo'

>>> a = A()
>>> a.foo
<bound method A.foo of <__main__.A object at 0x00A13EB0>>

>>> A.foo
<unbound method A.foo>

>>> A.__dict__['foo']
<function foo at 0x00A0A3F0>
>>> a.foo

>>> a.foo()
I am foo
>>> A.foo(a)
I am foo
>>> A.__dict__['foo'](a)
I am foo

class A(object):
    def foo():
        print 'I am foo'
    def foo2(cls):
        print 'I am foo2', cls 
    def foo3(self):
        print 'I am foo3', self       
    foo=staticmethod(foo)
    foo2=classmethod(foo2)

>>> a = A()
>>> a.foo()
I am foo

>>> A.foo()
I am foo

>>> A.foo2()
I am foo2 <class '__main__.A'>

>>> a.foo3()
I am foo3 <__main__.A object at 0x00A1AA10>

class M1(type): pass
class M2(M1):   pass

class C2(object): __metaclass__=M2    
class C1(object): __metaclass__=M1
class C3(C1, C2): pass

classes = [C1, C2, C3]
for c in classes:
    print c, c.__class__
    print '------------'                 

Output:

<class '__main__.C1'> <class '__main__.M1'>
------------
<class '__main__.C2'> <class '__main__.M2'>
------------
<class '__main__.C3'> <class '__main__.M2'>

class NoInit(object):
    def foo(self):
        self.x = 5

    def bar(self):
        print self.x

if __name__ == '__main__':            
    ni = NoInit()
    assert(not ni.__dict__.has_key('x'))
    try:
        ni.bar()
    except AttributeError, e:
        print e
    ni.foo()
    assert(ni.__dict__.has_key('x'))
    ni.bar()

Output:

'NoInit' object has no attribute 'x'
5

class A(object):
    def foo(self):
        print 3

if __name__ == '__main__':            
    a = A()
    f = a.foo
    print f
    print f.im_self
    a.foo()
    f()

Output:

<bound method A.foo of <__main__.A object at 0x00A03EB0>>
<__main__.A object at 0x00A03EB0>
3
3

import sys

class A(object):
    def __del__(self):
        print "That's it for me"

if __name__ == '__main__':            
    a = A()
    b = a
    print sys.getrefcount(a)-1
    del b
    print sys.getrefcount(a)-1

Output:

2
1
That's it for me

- __getattr__, __setattr__ and __getattribute__

class WriteOnlyDescriptor(object):
    def __init__(self):
        self.store = {}

    def __get__(self, obj, objtype=None):
        raise AttributeError 

    def __set__(self, obj, val):
        self.store[obj] = val

    def __del(self, obj):
        raise AttributeError

class A(object):
    password = WriteOnlyDescriptor()

if __name__ == '__main__': 
    a = A()
    try:
        print a.password
    except AttributeError, e:
        print e.__doc__
    a.password = 'secret'
    print A.__dict__['password'].store[a]

class A(object):
    def __init__(self):
        self.__password = None

    def getPassword(self):
        raise AttributeError

    def setPassword(self, password):        
        self.__password = password

    password = property(getPassword, setPassword)    

if __name__ == '__main__':
    a = A()
    try:
        print a.password
    except AttributeError, e:
        print e.__doc__
    a.password = 'secret'
    print a._A__password

Output:

Attribute not found.
secret 

return
def generate_ints(N):
    for i in range(N):
        yield i

	yield
yield
yield
return
yield
return
yield

	yield
yield
try

	yield
generate_ints
>>> gen = generate_ints(3)
>>> gen
<generator object at 0x8117f90>
>>> gen.next()
0
>>> gen.next()
1
>>> gen.next()
2
>>> gen.next()
Traceback (most recent call last):
  File "<stdin>", line 1, in ?
  File "<stdin>", line 2, in generate_ints
StopIteration

return

	return
StopIteration
next()
# A recursive generator that generates Tree leaves in in-order.
def inorder(t):
    if t:
        for x in inorder(t.left):
            yield x
        yield t.label
        for x in inorder(t.right):
            yield x

sentence := "Store it in the neighboring harbor"
if (i := find("or", sentence)) > 5 then write(i)

	find()
if
static int proc_sel(struct task_struct *p, int which, int who):

    if(p->pid): 

        switch (which): 

            case PRIO_PROCESS:

		if (!who && p == current): 

		return 1 

		return(p->pid == who) 

            case PRIO_PGRP:

		if (!who): 

		who = current->pgrp 

		return(p->pgrp == who) 

            case PRIO_USER:

		if (!who): 

		who = current->uid 

		return(p->uid == who) 

    return 0 

		(Modified from /linux/kernel/sys.c (C) Linus Torvalds) 

 The new generators in Python2.2 implement semi-coroutines, not
full coroutines:  the limitation is that they always return to
their caller -- they can't take an arbitrary continuation as
a return target.

 So you can't do everything you can do in Stackless, or at least,
you can't do it the same way. I'm not sure what the limitations
are yet, but you might be surprised with what you CAN do.
 If all the generator objects yield back to the same 'driver'
procedure, then you can do a sort of cooperative multithreading.
In that case, you're executing the generators for their side
effects -- the value returned by yield may be unimportant except
perhaps as a status code.

[ See also Tim Peters' post on the performance advantages of
  generators -- you only parse args once for many generator
  calls, and you keep and reuse the same stack frame. So I
  believe you get some of the same benefits of Stackless
  microthreads. ]

Maybe I can find the time to post an example of what I mean.
In the mean time, some previous posts on generators might give
you some ideas.

<http://groups.google.com/groups?q=generators+group:comp.lang.python+author:Majewski&hl=en&scoring=d&as_drrb=b&as_mind=12&as_minm=1&as_miny=2001&as_maxd=20&as_maxm=12&as_maxy=2001&rnum=1&selm=mailman.1008090197.6318.python-list%40python.org>

<http://groups.google.com/groups?hl=en&threadm=mailman.996870501.28562.python-list%40python.org&rnum=6&prev=/groups%3Fas_q%3Dgenerators%26as_ugroup%3Dcomp.lang.python%26as_uauthors%3DMajewski%26as_drrb%3Db%26as_mind%3D12%26as_minm%3D1%26as_miny%3D2001%26as_maxd%3D20%26as_maxm%3D12%26as_maxy%3D2001%26num%3D100%26as_scoring%3Dd%26hl%3Den>

The ungrammatical title of that last thread above:
"Nested generators is the Python equivalent of unix pipe cmds."
might also suggest the solution I'm thinking of.

-- Steve Majewski
__xxx__
[Alan spends the weekend upgrading an old laptop, so he can look at
this stuff in the comfort of the garden, which was bathed in glorious
sunshine for the entirety of the Irish Bank Holiday weekend B-). He
also prints and reads the writings of Christian Tismer[1], David
Mertz[2,3] and the Timbot[4,5], in relation to generators,
coroutines, continuations, pseudo/weightless/micro-threads, etc].

Steven Taschuk wrote:

[
 Lots of great stuff which illustrates the problems of calling from
 generator/coroutine code to functional code and vice versa.
]

I now understand. Thanks for taking the time to explain.

Thanks to your clear examples, I now picture coroutines in the
following way (which I hope is correct :-).

1. The "traditional" function/method call paradigm builds a stack
frame every time a function/method is called. Since this stack frame
holds the references to the local variables for the function/method,
it must be destroyed and collected if memory is not to be "leaked".
The only time when the stack frame can be destroyed is when the
references it contains are no longer required: i.e. after the
instance of the function/method call has finished.

2. The only method by which the "traditional" function call can
invoke another "traditional" function/method call is to call it
"recursively", thus causing the construction of another stack frame.
There is no mechanism for it to "substitute" the stack frame of the
called function "in place of" its own stack frame. (Although I
believe that Stackless can do this, after much rewriting and
reinvention of truth :-).

3. Because all "traditional" function/method calls, involve the
creation and eventual destruction of a stack frame, I label the space
in which they operate "Linear stack space".

4. There is another space, which I call "Generator space". When a
call is made into "Generator space", a stack frame is not
constructed: at least not every time. Instead, a resumable and
persistent stack frame is "resumed": this "resumable stack frame" was
created once, sometime in the past: it is termed, in current python
terminology, the generator-iterator.

5. When the code in "Generator space" (i.e. the generator-iterator)
is called, it resumes immediately after where it last 'yield'ed, and
continues until it 'yield's again, or returns or excepts. When it
'yield's, two things happen. A: The resumable stack frame is
"frozen", so that it can later be resumed again. and B: A python
object, which may be None, is transferred back to the caller.

6. For any call from "Linear Stack space" into "Generator space",
there is a crossover point, P. When the called code in "Generator
space" finishes executing, it can only enter back into "Linear stack
space" through point P: it cannot exit through any other point.
(According to the current python model).

7. If any calls are made from "Generator space" into "Linear stack
space", this leads to the creation of a stack frame, which must be
destroyed. If the called function/method in "Linear stack space" then
calls back into "Generator space", and the "Linear space" function is
not allowed to exit naturally, this is essentially unbound recursion,
which will lead eventually to a blown stack. (The non-functioning
Producer/Consumer example illustrates this: the code in "Generator
space" could be made to work by "traditionally" calling the write and
read functions, but this mutual recursion would eventually blow the
"Linear space" stack).

8. When a stack frame in "Generator space" 'yield's, it is possible
to adopt a convention for the 'yield'ed value: the "dispatcher"
convention. Under this convention, the value 'yield'ed can be a
reference to another resumable stack frame in Generator space. A
special "dispatch"-function is coded to recognise these references,
and to immediately call back into "Generator space". The newly
resumed stack frame is still limited by the same constraints as the
original call into "Generator space": it must eventually return to
"Linear stack space" through point P, in this case the dispatcher
function.

9. A complex network of interacting "resumable stack frames", or
generators, can mutually 'yield' control to each other, assuming the
cooperation of the dispatcher function. Execution continually
"bounces" back and forth between the dispatch function (Point P, in
"Linear stack space") and various 'yield'ed resumable states in
"Generator space".

10. A network of interacting generator-iterators could potentially
execute much faster than an equivalent series of "traditional"
function/method calls in "Linear stack space", since there is no
overhead associated with con/destruction of stack frames, no parsing of
parameters, etc. Such a network of interacting generators are called
"coroutines", and require the presence of a dispatcher function: the
dispatcher function must be explicitly created by the programmer, as
python currently exists.

10a: refinement: In fact python generators are really
"semi"-coroutines, because of the requirement to keep 'yield'ing to a
dispatcher function. In a language which implemented
"full"-coroutines (maybe a __future__ version of python?), the
dispatcher would not be required (at least explicitly), and resumable
states could transfer control directly to any other resumable state
for which they hold a reference.

11. The efficiency benefits of generators can be also realised by
*not* adopting the dispatcher convention. Instead, a generator can
simply 'yield' the series of values it has been coded to generate:
the nodes in a data structure, the numbers in a computed series, etc.
This can lead to more natural code, particularly for the calling
function which utilises the series of values 'yield'ed: it is mostly
unaware that there is a resumable state involved in the generation of
the values. Simple example:

def squares(n):
    for i in xrange(n):
        yield n, n*n

# Instantiate a generator-iterator
sqgen = squares(100)
# We can do the following because the generator-iterator
# conventiently implements the iterator protocol.
for i, sq in sqgen():
    print "The square of %d is %d" % (i, sq)

12. It is possible to avoid the "mutual" recursion problem of calling
from "Generator space" into "Linear stack space", by the following
actions

 o Moving all "traditional" function/method calls required from
   "Linear stack space" into "Generator space".
 o By expanding the "dispatcher" convention to allow generators to
   yield special values representing a function call or a return value.
   The dispatcher function must then explicitly manage a call stack.

13. It is possible to model ultra-lightweight threads by representing
each thread as a simple generator which steps through the states
required to implement the functionality required of the "thread". The
"execution pointer" of such threads is advanced simply by calling the
".next()" method of the "thread"s generator-iterator. (Incidentally,
as well as being highly efficient in terms of resource consumption,
these ultra-lightweight threads offer much finer control of
thread-prioritisation, thread-creation, destruction and "collection",
etc.)

Now that I think I've got my head around it, I think I'm going to try
my hand at an example or two, which I will post to the newsgroup
(might be a few weeks, I'm kinda busy right now). The two main
interesting examples that come to mind are

1. A ultra-lightweight thread implementation of an
   asyncore/medusa/twisted style socket server.
2. A coroutine based version of something that is normally
   (relatively) resource-intensive: a series of SAX2 callbacks to a
   chain of ContentHandlers/Filters.

Lastly, I think I'm beginning to "get" continuations. Although the
examples given in Christian Tismers paper "Continuations and
Stackless Python"[1] are difficult to follow without the definition
of a continuation object (which seems to require intimate familiarity
with the Stackless implementation), I think I understand the general
principle.

And it's a principal I'm not really interested in pursuing, because I
can't see that it will make me any more productive, or my programs
more efficient (than they could be using the "coroutines" and
"weightless-threads" described above). And I can imagine that
continuation code could be a little brain-bending to write (thus
making me *less* productive %-), as this example of a while loop
(which I sort of understand) shows:

def looptest(n):
    this = continuation.current()
    k = this.update(n)
    if k:
        this(k-1)
    else:
        del this.link

But I can see the power inherent in the continuations paradigm.

Again, many thanks to all who responded to my original post.

Reading material.

[1] http://www.stackless.com/spcpaper.pdf
[2] http://www-106.ibm.com/developerworks/linux/library/l-pygen.html
[3] http://www-106.ibm.com/developerworks/library/l-pythrd.html
[4] http://tinyurl.com/dbyn
[5] http://www.tismer.com/research/stackless/coroutines.tim.peters.html

kind regards,

--
alan kennedy

"John Roth" <johnroth@ameritech.net> wrote in message
news:uv94kkq5ajbae3@news.supernews.com...
>
> "Rocco Moretti" <roccomoretti@netscape.net> wrote
> > process of set attributes -> call processing function -> repeat is an
> > akward way of achieving what you really want to do - continue the
> > generator with given data. I'd agree with what Guido said on
> > Python-Dev: there is no advantage in this scheme over passing a
> > mutable dummy object when the generator is created (you could even
> > call it __self__ if you really wanted too ...). 
> I thoroughly agree.

Me too.

> with some bemusement, because, for me at least, the most
> intuitive thing to do is make yield a built in function rather than
> a statement.

But then we would need a another keyword (possible, of course) to tell
the compiler to change the generated bytecode.  It would also break
the parallel between 'return something' and 'yield something'.  The
only difference in execution is that yield does less -- by *not*
deleting the execution frame.

> That way, it can return a value, which is what
> was passed in on the reinvocation.

One of the design features of generators is that resumption is
extremely quick precisely because the argument processing and function
setup steps are bypassed.  Just restore the execution frame and go on
with the next statement [bytecode actually].

Another design feature is that they are meant to work seemlessly with
for statements, with only one explicit call (to produce the iterator
that 'for' needs).  In this context, passing in additional values is
not possible.

Another problem is with multiple yields.  Consider the following
hypothetical generator with the proposed yield() 'function' (see next
comment for why the '' marks):

def genf(a):
  b=yield(a)
  b,c=yield(a+b)
  yield(a+b+c)
gen = genf(1)

Then the documentation must say: on the first call to gen.next(), pass
one arg; on the next, pass two; finally, don't pass any.  Not good,
methings.

> The trouble with the whole thing is that conceptually, "yield" has two
> values: one that it returns to the caller, and one that it
> gets from the caller. It's a two way pipe.

(You mean, 'would be'.)  Pipes and such usually have explicit read and
write methods that make the order of operations clear.  The yield()
proposal hijacks function notation to squash a read and write
together -- with the order switched at the two ends.

To explain: y=f(x) usually means 'set y to a value depending on
(calculated from) x'  -- this is the meaning of 'function'.
Operationally, a function call means to send x to process f to
initialize the corresponding parameter,  suspend operation while f
operates, and set y to the value returned by f.  The combined
next(arg)/yield() idea warps and shuffles these semantics.

Let p = pipe or whatever:  Then (omitting suspend) y = yield(x) could
mean
  p.write(x); y=p.read()
where (contrary to the implication of function notation) the value
read cannot depend on x since it is calculated at the other end before
the write.

The corresponding x = gen.next(y) then means
  x=p.read; p.write(y)
which makes the order of value passing the opposite of what a function
call means.  The only reason to write y is for a possible future call
to gen.next().  I think it much better to separate the read and write
and pair the writing of y with the reading of the x that functionally
depends on the value written.

One could reverse the meaning of x = gen.next(y) to be
  p.write(y); x=p.read()
but the x read would still not depend on the y written since the
latter would have to be set aside and ignored until the predetermined
x was sent back.  IE, y=yield(x) would have to mean and be implemented
as
  <hidden-slot> = p.read(); p.write(x); y=<hidden-slot>

In either case, there is a synchronization problem in that the values
sent to the generator are shifted by one call.  The last gen.next()
call must send a dummy that will be ignored.  On the other hand, if,
for instance, there is one yield function which depends on the
variable reset by the yield function, then that variable must be
separately initialized in the initial genf() call.

So it arguably makes just as much sense to initialize via genf() with
a mutable with paired write/read, send/receive, or set/get methods or
the equivalent operations (as with dicts).  One can then make explicit
calls to pass data in either direction without fake 'function' calls.
If one pairs 'yield None' with 'dummy=gen.next()' or 'for dummy in
genf(mutable):', one can even do all value passing, in both
directions, via the two-way channel or mutable and use 'yield'
strictly for suspend/resume synchronization of the coroutines.

--
This proposal come close to asking for what the original Stackless did
with continuations.  These allowed some mind-boggling code.  Almost
too fancy for what Python is meant to be 8-).

Terry J. Reedy
Major revision:  more details about exceptions, return vs StopIteration, and
interactions with try/except/finally; more Q&A; and a BDFL Pronouncement.  The
reference implementation appears solid and works as described here in all
respects, so I expect this will be the last major revision (and so also last
full posting) of this PEP.

The output below is in ndiff format (see Tools/scripts/ndiff.py in your Python
distribution).  Just the new text can be seen in HTML form here:

    http://python.sf.net/peps/pep-0255.html

"Feature discussions" should take place primarily on the Python Iterators list:

    mailto:python-iterators@lists.sourceforge.net

Implementation discussions may wander in and out of Python-Dev too.

  PEP: 255
  Title: Simple Generators
- Version: $Revision: 1.3 $
?                       ^
+ Version: $Revision: 1.12 $
?                       ^^
  Author: nas@python.ca (Neil Schemenauer),
      tim.one@home.com (Tim Peters),
      magnus@hetland.org (Magnus Lie Hetland)
  Discussion-To: python-iterators@lists.sourceforge.net
  Status: Draft
  Type: Standards Track
  Requires: 234
  Created: 18-May-2001
  Python-Version: 2.2
- Post-History: 14-Jun-2001
+ Post-History: 14-Jun-2001, 23-Jun-2001
?                          +++++++++++++

  Abstract

      This PEP introduces the concept of generators to Python, as well
      as a new statement used in conjunction with them, the "yield"
      statement.

  Motivation

      When a producer function has a hard enough job that it requires
      maintaining state between values produced, most programming languages
      offer no pleasant and efficient solution beyond adding a callback
      function to the producer's argument list, to be called with each value
      produced.

      For example, tokenize.py in the standard library takes this approach:
      the caller must pass a "tokeneater" function to tokenize(), called
      whenever tokenize() finds the next token.  This allows tokenize to be
      coded in a natural way, but programs calling tokenize are typically
      convoluted by the need to remember between callbacks which token(s)
      were seen last.  The tokeneater function in tabnanny.py is a good
      example of that, maintaining a state machine in global variables, to
      remember across callbacks what it has already seen and what it hopes to
      see next.  This was difficult to get working correctly, and is still
      difficult for people to understand.  Unfortunately, that's typical of
      this approach.

      An alternative would have been for tokenize to produce an entire parse
      of the Python program at once, in a large list.  Then tokenize clients
      could be written in a natural way, using local variables and local
      control flow (such as loops and nested if statements) to keep track of
      their state.  But this isn't practical:  programs can be very large, so
      no a priori bound can be placed on the memory needed to materialize the
      whole parse; and some tokenize clients only want to see whether
      something specific appears early in the program (e.g., a future
      statement, or, as is done in IDLE, just the first indented statement),
      and then parsing the whole program first is a severe waste of time.

      Another alternative would be to make tokenize an iterator[1],
      delivering the next token whenever its .next() method is invoked.  This
      is pleasant for the caller in the same way a large list of results
      would be, but without the memory and "what if I want to get out early?"
      drawbacks.  However, this shifts the burden on tokenize to remember
      *its* state between .next() invocations, and the reader need only
      glance at tokenize.tokenize_loop() to realize what a horrid chore that
      would be.  Or picture a recursive algorithm for producing the nodes of
      a general tree structure:  to cast that into an iterator framework
      requires removing the recursion manually and maintaining the state of
      the traversal by hand.

      A fourth option is to run the producer and consumer in separate
      threads.  This allows both to maintain their states in natural ways,
      and so is pleasant for both.  Indeed, Demo/threads/Generator.py in the
      Python source distribution provides a usable synchronized-communication
      class for doing that in a general way.  This doesn't work on platforms
      without threads, though, and is very slow on platforms that do
      (compared to what is achievable without threads).

      A final option is to use the Stackless[2][3] variant implementation of
      Python instead, which supports lightweight coroutines.  This has much
      the same programmatic benefits as the thread option, but is much more
      efficient.  However, Stackless is a controversial rethinking of the
      Python core, and it may not be possible for Jython to implement the
      same semantics.  This PEP isn't the place to debate that, so suffice it
      to say here that generators provide a useful subset of Stackless
      functionality in a way that fits easily into the current CPython
      implementation, and is believed to be relatively straightforward for
      other Python implementations.

      That exhausts the current alternatives.  Some other high-level
      languages provide pleasant solutions, notably iterators in Sather[4],
      which were inspired by iterators in CLU; and generators in Icon[5], a
      novel language where every expression "is a generator".  There are
      differences among these, but the basic idea is the same:  provide a
      kind of function that can return an intermediate result ("the next
      value") to its caller, but maintaining the function's local state so
      that the function can be resumed again right where it left off.  A
      very simple example:

          def fib():
              a, b = 0, 1
              while 1:
                  yield b
                  a, b = b, a+b

      When fib() is first invoked, it sets a to 0 and b to 1, then yields b
      back to its caller.  The caller sees 1.  When fib is resumed, from its
      point of view the yield statement is really the same as, say, a print
      statement:  fib continues after the yield with all local state intact.
      a and b then become 1 and 1, and fib loops back to the yield, yielding
      1 to its invoker.  And so on.  From fib's point of view it's just
      delivering a sequence of results, as if via callback.  But from its
      caller's point of view, the fib invocation is an iterable object that
      can be resumed at will.  As in the thread approach, this allows both
      sides to be coded in the most natural ways; but unlike the thread
      approach, this can be done efficiently and on all platforms.  Indeed,
      resuming a generator should be no more expensive than a function call.

      The same kind of approach applies to many producer/consumer functions.
      For example, tokenize.py could yield the next token instead of invoking
      a callback function with it as argument, and tokenize clients could
      iterate over the tokens in a natural way:  a Python generator is a kind
      of Python iterator[1], but of an especially powerful kind.

- Specification
+ Specification:  Yield
?              ++++++++

      A new statement is introduced:

          yield_stmt:    "yield" expression_list

      "yield" is a new keyword, so a future statement[8] is needed to phase
-     this in.  [XXX spell this out]
+     this in.  [XXX spell this out -- but new keywords have ripple effects
+     across tools too, and it's not clear this can be forced into the future
+     framework at all -- it's not even clear that Python's parser alone can
+     be taught to swing both ways based on a future stmt]

      The yield statement may only be used inside functions.  A function that
-     contains a yield statement is called a generator function.
+     contains a yield statement is called a generator function.  A generator
?                                                               +++++++++++++
+     function is an ordinary function object in all respects, but has the
+     new CO_GENERATOR flag set in the code object's co_flags member.

      When a generator function is called, the actual arguments are bound to
      function-local formal argument names in the usual way, but no code in
      the body of the function is executed.  Instead a generator-iterator
      object is returned; this conforms to the iterator protocol[6], so in
      particular can be used in for-loops in a natural way.  Note that when
      the intent is clear from context, the unqualified name "generator" may
      be used to refer either to a generator-function or a generator-
      iterator.

      Each time the .next() method of a generator-iterator is invoked, the
      code in the body of the generator-function is executed until a yield
      or return statement (see below) is encountered, or until the end of
      the body is reached.

      If a yield statement is encountered, the state of the function is
      frozen, and the value of expression_list is returned to .next()'s
      caller.  By "frozen" we mean that all local state is retained,
      including the current bindings of local variables, the instruction
      pointer, and the internal evaluation stack:  enough information is
      saved so that the next time .next() is invoked, the function can
      proceed exactly as if the yield statement were just another external
      call.

+     Restriction:  A yield statement is not allowed in the try clause of a
+     try/finally construct.  The difficulty is that there's no guarantee
+     the generator will ever be resumed, hence no guarantee that the finally
+     block will ever get executed; that's too much a violation of finally's
+     purpose to bear.
+
+
+ Specification:  Return
+
      A generator function can also contain return statements of the form:

          "return"

      Note that an expression_list is not allowed on return statements
      in the body of a generator (although, of course, they may appear in
      the bodies of non-generator functions nested within the generator).

-     When a return statement is encountered, nothing is returned, but a
+     When a return statement is encountered, control proceeds as in any
+     function return, executing the appropriate finally clauses (if any
-     StopIteration exception is raised, signalling that the iterator is
?                                                           ------------
+     exist).  Then a StopIteration exception is raised, signalling that the
?    ++++++++++++++++
-     exhausted.   The same is true if control flows off the end of the
+     iterator is exhausted.   A StopIteration exception is also raised if
+     control flows off the end of the generator without an explict return.
+
-     function.  Note that return means "I'm done, and have nothing
?   -----------
+     Note that return means "I'm done, and have nothing interesting to
?                                                       +++++++++++++++
-     interesting to return", for both generator functions and non-generator
?    ---------------
+     return", for both generator functions and non-generator functions.
?                                                            +++++++++++
-     functions.
+
+     Note that return isn't always equivalent to raising StopIteration:  the
+     difference lies in how enclosing try/except constructs are treated.
+     For example,
+
+         >>> def f1():
+         ...     try:
+         ...         return
+         ...     except:
+         ...        yield 1
+         >>> print list(f1())
+         []
+
+     because, as in any function, return simply exits, but
+
+         >>> def f2():
+         ...     try:
+         ...         raise StopIteration
+         ...     except:
+         ...         yield 42
+         >>> print list(f2())
+         [42]
+
+     because StopIteration is captured by a bare "except", as is any
+     exception.
+
+
+ Specification:  Generators and Exception Propagation
+
+     If an unhandled exception-- including, but not limited to,
+     StopIteration --is raised by, or passes through, a generator function,
+     then the exception is passed on to the caller in the usual way, and
+     subsequent attempts to resume the generator function raise
+     StopIteration.  In other words, an unhandled exception terminates a
+     generator's useful life.
+
+     Example (not idiomatic but to illustrate the point):
+
+     >>> def f():
+     ...     return 1/0
+     >>> def g():
+     ...     yield f()  # the zero division exception propagates
+     ...     yield 42   # and we'll never get here
+     >>> k = g()
+     >>> k.next()
+     Traceback (most recent call last):
+       File "<stdin>", line 1, in ?
+       File "<stdin>", line 2, in g
+       File "<stdin>", line 2, in f
+     ZeroDivisionError: integer division or modulo by zero
+     >>> k.next()  # and the generator cannot be resumed
+     Traceback (most recent call last):
+       File "<stdin>", line 1, in ?
+     StopIteration
+     >>>
+
+
+ Specification:  Try/Except/Finally
+
+     As noted earlier, yield is not allowed in the try clause of a try/
+     finally construct.  A consequence is that generators should allocate
+     critical resources with great care.  There is no restriction on yield
+     otherwise appearing in finally clauses, except clauses, or in the try
+     clause of a try/except construct:
+
+     >>> def f():
+     ...     try:
+     ...         yield 1
+     ...         try:
+     ...             yield 2
+     ...             1/0
+     ...             yield 3  # never get here
+     ...         except ZeroDivisionError:
+     ...             yield 4
+     ...             yield 5
+     ...             raise
+     ...         except:
+     ...             yield 6
+     ...         yield 7     # the "raise" above stops this
+     ...     except:
+     ...         yield 8
+     ...     yield 9
+     ...     try:
+     ...         x = 12
+     ...     finally:
+     ...         yield 10
+     ...     yield 11
+     >>> print list(f())
+     [1, 2, 4, 5, 8, 9, 10, 11]
+     >>>

  Example

          # A binary tree class.
          class Tree:

              def __init__(self, label, left=None, right=None):
                  self.label = label
                  self.left = left
                  self.right = right

              def __repr__(self, level=0, indent="    "):
                  s = level*indent + `self.label`
                  if self.left:
                      s = s + "\n" + self.left.__repr__(level+1, indent)
                  if self.right:
                      s = s + "\n" + self.right.__repr__(level+1, indent)
                  return s

              def __iter__(self):
                  return inorder(self)

          # Create a Tree from a list.
          def tree(list):
              n = len(list)
              if n == 0:
                  return []
              i = n / 2
              return Tree(list[i], tree(list[:i]), tree(list[i+1:]))

          # A recursive generator that generates Tree leaves in in-order.
          def inorder(t):
              if t:
                  for x in inorder(t.left):
                      yield x
                  yield t.label
                  for x in inorder(t.right):
                      yield x

          # Show it off: create a tree.
          t = tree("ABCDEFGHIJKLMNOPQRSTUVWXYZ")
          # Print the nodes of the tree in in-order.
          for x in t:
              print x,
          print

          # A non-recursive generator.
          def inorder(node):
              stack = []
              while node:
                  while node.left:
                      stack.append(node)
                      node = node.left
                  yield node.label
                  while not node.right:
                      try:
                          node = stack.pop()
                      except IndexError:
                          return
                      yield node.label
                  node = node.right

          # Exercise the non-recursive generator.
          for x in t:
              print x,
          print

+     Both output blocks display:
+
+         A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
+

  Q & A

+     Q. Why not a new keyword instead of reusing "def"?
+
+     A. See BDFL Pronouncements section below.
+
-     Q. Why a new keyword?  Why not a builtin function instead?
+     Q. Why a new keyword for "yield"?  Why not a builtin function instead?
?                         ++++++++++++

      A. Control flow is much better expressed via keyword in Python, and
         yield is a control construct.  It's also believed that efficient
         implementation in Jython requires that the compiler be able to
         determine potential suspension points at compile-time, and a new
-        keyword makes that easy.
+        keyword makes that easy.  The CPython referrence implementation also
+        exploits it heavily, to detect which functions *are* generator-
+        functions (although a new keyword in place of "def" would solve that
+        for CPython -- but people asking the "why a new keyword?" question
+        don't want any new keyword).
+
+     Q: Then why not some other special syntax without a new keyword?  For
+        example, one of these instead of "yield 3":
+
+        return 3 and continue
+        return and continue 3
+        return generating 3
+        continue return 3
+        return >> , 3
+        from generator return 3
+        return >> 3
+        return << 3
+        >> 3
+        << 3
+
+     A: Did I miss one <wink>?  Out of hundreds of messages, I counted two
+        suggesting such an alternative, and extracted the above from them.
+        It would be nice not to need a new keyword, but nicer to make yield
+        very clear -- I don't want to have to *deduce* that a yield is
+        occurring from making sense of a previously senseless sequence of
+        keywords or operators.  Still, if this attracts enough interest,
+        proponents should settle on a single consensus suggestion, and Guido
+        will Pronounce on it.
+
+     Q. Why allow "return" at all?  Why not force termination to be spelled
+        "raise StopIteration"?
+
+     A. The mechanics of StopIteration are low-level details, much like the
+        mechanics of IndexError in Python 2.1:  the implementation needs to
+        do *something* well-defined under the covers, and Python exposes
+        these mechanisms for advanced users.  That's not an argument for
+        forcing everyone to work at that level, though.  "return" means "I'm
+        done" in any kind of function, and that's easy to explain and to use.
+        Note that "return" isn't always equivalent to "raise StopIteration"
+        in try/except construct, either (see the "Specification: Return"
+        section).
+
+     Q. Then why not allow an expression on "return" too?
+
+     A. Perhaps we will someday.  In Icon, "return expr" means both "I'm
+        done", and "but I have one final useful value to return too, and
+        this is it".  At the start, and in the absence of compelling uses
+        for "return expr", it's simply cleaner to use "yield" exclusively
+        for delivering values.
+
+
+ BDFL Pronouncements
+
+     Issue:  Introduce another new keyword (say, "gen" or "generator") in
+     place of "def", or otherwise alter the syntax, to distinguish
+     generator-functions from non-generator functions.
+
+     Con:  In practice (how you think about them), generators *are*
+     functions, but with the twist that they're resumable.  The mechanics of
+     how they're set up is a comparatively minor technical issue, and
+     introducing a new keyword would unhelpfully overemphasize the
+     mechanics of how generators get started (a vital but tiny part of a
+     generator's life).
+
+     Pro:  In reality (how you think about them), generator-functions are
+     actually factory functions that produce generator-iterators as if by
+     magic.  In this respect they're radically different from non-generator
+     functions, acting more like a constructor than a function, so reusing
+     "def" is at best confusing.  A "yield" statement buried in the body is
+     not enough warning that the semantics are so different.
+
+     BDFL:  "def" it stays.  No argument on either side is totally
+     convincing, so I have consulted my language designer's intuition.  It
+     tells me that the syntax proposed in the PEP is exactly right - not too
+     hot, not too cold.  But, like the Oracle at Delphi in Greek mythology,
+     it doesn't tell me why, so I don't have a rebuttal for the arguments
+     against the PEP syntax.  The best I can come up with (apart from
+     agreeing with the rebuttals ... already made) is "FUD".  If this had
+     been part of the language from day one, I very much doubt it would have
+     made Andrew Kuchling's "Python Warts" page.

  Reference Implementation

-     A preliminary patch against the CVS Python source is available[7].
+     The current implementation, in a preliminary state (no docs and no
+     focused tests), is part of Python's CVS development tree[9].
+     Using this requires that you build Python from source.
+
+     This was derived from an earlier patch by Neil Schemenauer[7].

  Footnotes and References

      [1] PEP 234, http://python.sf.net/peps/pep-0234.html
      [2] http://www.stackless.com/
      [3] PEP 219, http://python.sf.net/peps/pep-0219.html
      [4] "Iteration Abstraction in Sather"
          Murer , Omohundro, Stoutamire and Szyperski
          http://www.icsi.berkeley.edu/~sather/Publications/toplas.html
      [5] http://www.cs.arizona.edu/icon/
      [6] The concept of iterators is described in PEP 234
          http://python.sf.net/peps/pep-0234.html
      [7] http://python.ca/nas/python/generator.diff
      [8] http://python.sf.net/peps/pep-0236.html
+     [9] To experiment with this implementation, check out Python from CVS
+         according to the instructions at
+             http://sf.net/cvs/?group_id=5470

  Copyright

      This document has been placed in the public domain.

  Local Variables:
  mode: indented-text
  indent-tabs-mode: nil
  End:
[Neel Krishnaswami]
> ...
> I've been looking at Icon, and it occurs to me that if coroutines and
> generators were available at the Python level, it might yield a way of
> doing string processing that is more "Pythonic" than regexps.

Yes, and you can find much more about this in the very early days of the
String SIG archives.

> Regexps are nice because when you have a pattern that they can
> directly represent, then you can simply specify a pattern and then you
> don't have to worry about the tedious bookkeeping of looping over the
> string and keeping track of the state variable.
>
> However, when you need to match a pattern that a regexp can't match,
> then suddenly you need to break the string into tokens with regexps
> and then loop over the pieces and keep track of a bunch of state
> variables that don't necessarily correspond to the pieces you are
> actually interested in.
>
> This is unpleasant, because a) clumsy bookkeeping is bad, and b)
> there's two different sorts of syntax needed to do basically
> similar tasks.

The latter is one of the arguments Griswold gave in the paper that
introduced Icon, contrasting Icon's uniform approach to SNOBOL4's distinct
pattern and procedural sublanguages.

I don't think he'd make the same argument today!  Icon is an idiosyncratic
delight, but most string pattern matching was in fact easier (to write,
modify, or read) in SNOBOL4.  Once you start using Icon for complex pattern
matching, you'll soon discover that it's very hard to go back to old code
and disentangle "the pattern" from "the processing" -- *because* everything
looks the same, and it's all intermixed.  The distinct sublanguages in
SNOBOL4 enforced a clean separation; and that was more often an aid than a
burden.

Have noted before that writing string matching code in Icon feels a lot like
writing in an assembly language for SNOBOL4.  Of course the latter didn't
have Icon's power in other areas (e.g. it's at least possible to program
structural pattern-matchers in Icon, using generators to match e.g. tree
patterns; SNOBOL4's patterns started & ended with strings).

> If we could compose generators just like functions, then the bookkeeping
> can be abstracted away and the same approach will work for arbitrarily
> complicated parsing tasks.

The real advantage of regexps is speed, & that's probably while they'll
always be much more popular.  SNOBOL4 and Icon didn't define "bal" builtins
because you couldn't code that pattern yourself <wink>.  bal is beyond a
regexp's abilities, but it's still a simple kind of pattern, and just runs
too darned slow if implemented via the recursive definition as run with the
general backtracking machinery.

> So I think it would be nice if these lovely toys were available at
> the Python level. Is this a possibility?

I've been bugging Guido about generators since '91, but for the billion and
one uses other than string matching.  Anything is possible -- except that
I'll ever stop bugging him <wink>.

> (I'd love to be able to define coroutines in Python like so:
>
> def evens(z):
>     for elt in z:
>         if z % 2 == 0:
>             suspend(z)

How do you intend to resume it?

> It would probably require some rethinking of Python's iteration
> protocol though.)

That's not a hangup; Guido already knows how to do it cleanly.  Like any Big
New Feature there are real questions about costs vs benefits, and so far
this stuff has been widely viewed as "too esoteric".  I think it's natural
as breathing, to the point that a *non*-resumable function strikes me as
never inhaling <wink>.

For now, there's a working implementation of a generator protocol (somewhat
more flexible than Icon's) in the source distribution, under
Demo/threads/Generator.py.  I also posted a general (thread-based) coroutine
implementation a bit over 5 years ago.  Building these on Christian's
stackless Python instead should run much faster.

BTW, generators are much easier to implement than coroutines -- the former
don't require threads or stacklessness or continuations under the covers,
just some relatively straightforward changes to Python's current
implementation (Steven Majewski noted this 5 years ago).  This is why
generators work in all ports of Icon, but coroutines are an optional feature
that's supported only if a platform guru has taken the time to write the
platform-dependent context-switching assembly code Icon coexps require.

degeneratoredly y'rs  - tim
Aaron Watters wrote:
> 
> James C. Phillips wrote:
> >
> > Daniel Larsson wrote:
> > > Coroutines are still rare in newer languages. I'm sure there
> > > are more, but I know only of Modula-2 and BETA.
> >
> > For us non-computer-science-types, what is a coroutine?
> > I've never heard this term before.
> >
> > -Jim
> 
> I've never really used them, but as I recall they are like
> subroutines, except that more than one coroutine can be
> active at once and one coroutine can explicitly give control
> to another or something... I personally don't understand why
> you need programming language features to emulate this kind of
> behaviour, but I'm probably ill informed and wrong.
> 
> Maybe some expert can tell me: is there anything you can do
> with coroutines that can't be emulated directly with instances
> of classes in Python or M3?  Please educate...  -- Aaron Watters

I would hardly describe myself as an expert in coroutines, but
anyway...

Suppose you have a binary tree class:

class Node:
   def __init__(self, elem, left=None, right=None):
      self.elem = elem
      self.left, self.right = left, right

class BinTree:
   def __init__(self):
      self.root = None

   def scan(self, node):
      if node:
         self.scan(node.left)
	 suspend node.elem # Suspend coroutine and return value
         self.scan(node.right)

   coroutine traverse(self):
      self.scan(self.root)
      return None # Terminate coroutine

A coroutine has its own stack. When a coroutine is called, the
caller's stack is saved, and the execution is moved to the callee's
stack. When a coroutine suspends, stacks are reversed again.

In the above case, each call to traverse will do one inorder step
in the tree and return the value at that node. Here's how to use
it to print the contents of a binary trees:

tree = BinTree()
... # init tree
while 1:
   elem = tree.traverse()
   if elem == None: break
   print elem

This could certainly be implemented in Python without coroutines,
and I don't even know if this is even a good example of the
benefits of coroutines. Anyway, some problems where you would like
to use threads, might be easier to solve with coroutines, since
you don't have any synchronization problems (you have explicit
control of when switching between threads).

Hope I didn't confuse too many people out there...
-- 
Daniel Larsson, ABB Industrial Systems AB

Howdy,

Andrew Cooke wrote:
> 
> In article <000001bf768e$48e40580$45a0143f@tim>,
>   "Tim Peters" <tim_one@email.msn.com> wrote:
> >    def traverse_post(self):
> >        for child in self.left, self.right:
> >            if child is not None:
> >                suspend child.traverse_post()
> >        suspend self.data
> 
> That finally hammered home to me just what continuations are all about.
> Does anyone have something similarly elegant that shows a coroutine?
> I've just had a look at the stackless python documentation and
> coroutines seem to be described as something coming out of a single
> detailed example.  Is there a simpler definition?  (I did look back
> through some posts on Deja, but there was nothing recent that seemed to
> explain what a coroutine actually is - sorry if I've missed something
> obvious).

What did you read, actually?
Here some pointers:
Homepage:   http://www.tismer.com/research/stackless/
IPC8 paper: http://www.tismer.com/research/stackless/spcpaper.htm
Slides:     http://www.tismer.com/research/stackless/spc-sheets.ppt
The latter gives you a bit of understanding what a continuation is.

Tim Peters about coroutines can be found here:
http://www.tismer.com/research/stackless/coroutines.tim.peters.html

More on coroutines by Sam Rushing:
http://www.nightmare.com/~rushing/copython/index.html

On Scheme, continuations, generators and coroutines:
http://www.cs.rice.edu/~dorai/t-y-scheme/

Revised Scheme 5 report:
http://www.swiss.ai.mit.edu/~jaffer/r5rs_toc.html

The following books are also highly recommended:
- Andrew W. Appel, Compiling with Continuations, Cambridge University
Press, 1992
- Daniel P. Friedman, Mitchell Wand, and Christopher T. Haynes,
Essentials of Programming Languages, MIT Press, 1993
- Christopher T. Haynes, Daniel P. Friedman, and Mitchell Wand,
Continuations and Coroutines,  Computer Languages, 11(3/4): 143-153,
1986.
- Strachey and Wadsworth, Continuations: A mathematical semantics which
can deal with full jumps.  Technical monograph PRG-11, Programming
Research Group, Oxford, 1974.

I don't think this is easy stuff at all, and you can't expect
to find a simple answer by skimming a couple of web pages.
It took me a lot of time to understand a fair part of this,
and this is also a showstopper to get this stuff to be used.

> Also, comp.lang.lisp is currently dissing continuations.  Would that be
> because the alternative is to pass the code that will process the node
> into the iterator as a (first class) function?  Obviously, in this case,
> yes, but is that true in general (are there examples where continuations
> or coroutines make something possible that really is tricky to do in
> other ways)?

An iterator is quite an easy thing, and it can be implemented
without continuations rather easily. Continuations cover problems
of another order of magnitude. It is due to too simple examples
that everybody thinks that coroutines and generators are the
only target. Continuations can express any kind of control flow,
and they can model iterators, coroutines and generators easily.
The opposite is not true!

I know this isn't enough to convince you, but at the time I have
no chance. I need to build some very good demo applications
which use continuations without exposing them to the user,
and this is admittedly difficult.

ciao - chris

-- 
Christian Tismer             :^)   <mailto:tismer@appliedbiometrics.com>
Applied Biometrics GmbH      :     Have a break! Take a ride on Python's
Düppelstr. 31                :    *Starship* http://starship.python.net
12163 Berlin                 :     PGP key -> http://wwwkeys.pgp.net
PGP Fingerprint       E182 71C7 1A9D 66E9 9D15  D3CC D4D7 93E2 1FAE F6DF
     we're tired of banana software - shipped green, ripens at home

(coroutine (lambda (v) 
  <body>))
(define Coroutine-1 (coroutine (lambda (v) ...)))

(resume 
  <co> <val>)
<co>
v
(resume x B)
x
v
(resume y ..)
x
resume
This PEP discusses changes required to core Python in order to
    efficiently support generators, microthreads and coroutines. It is
    related to PEP 220, which describes how Python should be extended
    to support these facilities. The focus of this PEP is strictly on
    the changes required to allow these extensions to work.

    While these PEPs are based on Christian Tismer's Stackless[1]
    implementation, they do not regard Stackless as a reference
    implementation.  Stackless (with an extension module) implements
    continuations, and from continuations one can implement
    coroutines, microthreads (as has been done by Will Ware[2]) and
    generators. But in more that a year, no one has found any other
    productive use of continuations, so there seems to be no demand
    for their support.

    However, Stackless support for continuations is a relatively minor
    piece of the implementation, so one might regard it as "a"
    reference implementation (rather than "the" reference
    implementation).

    Generators and coroutines have been implemented in a number of
    languages in a number of ways. Indeed, Tim Peters has done pure
    Python implementations of generators[3] and coroutines[4] using
    threads (and a thread-based coroutine implementation exists for
    Java). However, the horrendous overhead of a thread-based
    implementation severely limits the usefulness of this approach.

    Microthreads (a.k.a "green" or "user" threads) and coroutines
    involve transfers of control that are difficult to accommodate in
    a language implementation based on a single stack. (Generators can
    be done on a single stack, but they can also be regarded as a very
    simple case of coroutines.)

    Real threads allocate a full-sized stack for each thread of
    control, and this is the major source of overhead. However,
    coroutines and microthreads can be implemented in Python in a way
    that involves almost no overhead.  This PEP, therefor, offers a
    way for making Python able to realistically manage thousands of
    separate "threads" of activity (vs. todays limit of perhaps dozens
    of separate threads of activity).

    Another justification for this PEP (explored in PEP 220) is that
    coroutines and generators often allow a more direct expression of
    an algorithm than is possible in today's Python.

    The first thing to note is that Python, while it mingles
    interpreter data (normal C stack usage) with Python data (the
    state of the interpreted program) on the stack, the two are
    logically separate. They just happen to use the same stack.

    A real thread gets something approaching a process-sized stack
    because the implementation has no way of knowing how much stack
    space the thread will require. The stack space required for an
    individual frame is likely to be reasonable, but stack switching
    is an arcane and non-portable process, not supported by C.

    Once Python stops putting Python data on the C stack, however,
    stack switching becomes easy.

    The fundamental approach of the PEP is based on these two
    ideas. First, separate C's stack usage from Python's stack
    usage. Secondly, associate with each frame enough stack space to
    handle that frame's execution.

    In the normal usage, Stackless Python has a normal stack
    structure, except that it is broken into chunks. But in the
    presence of a coroutine / microthread extension, this same
    mechanism supports a stack with a tree structure.  That is, an
    extension can support transfers of control between frames outside
    the normal "call / return" path.

    The major difficulty with this approach is C calling Python. The
    problem is that the C stack now holds a nested execution of the
    byte-code interpreter. In that situation, a coroutine /
    microthread extension cannot be permitted to transfer control to a
    frame in a different invocation of the byte-code interpreter. If a
    frame were to complete and exit back to C from the wrong
    interpreter, the C stack could be trashed.

    The ideal solution is to create a mechanism where nested
    executions of the byte code interpreter are never needed. The easy
    solution is for the coroutine / microthread extension(s) to
    recognize the situation and refuse to allow transfers outside the
    current invocation.

    We can categorize code that involves C calling Python into two
    camps: Python's implementation, and C extensions. And hopefully we
    can offer a compromise: Python's internal usage (and C extension
    writers who want to go to the effort) will no longer use a nested
    invocation of the interpreter. Extensions which do not go to the
    effort will still be safe, but will not play well with coroutines
    / microthreads.

    Generally, when a recursive call is transformed into a loop, a bit
    of extra bookkeeping is required. The loop will need to keep it's
    own "stack" of arguments and results since the real stack can now
    only hold the most recent. The code will be more verbose, because
    it's not quite as obvious when we're done. While Stackless is not
    implemented this way, it has to deal with the same issues.

    In normal Python, PyEval_EvalCode is used to build a frame and
    execute it. Stackless Python introduces the concept of a
    FrameDispatcher. Like PyEval_EvalCode, it executes one frame. But
    the interpreter may signal the FrameDispatcher that a new frame
    has been swapped in, and the new frame should be executed. When a
    frame completes, the FrameDispatcher follows the back pointer to
    resume the "calling" frame.

    So Stackless transforms recursions into a loop, but it is not the
    FrameDispatcher that manages the frames. This is done by the
    interpreter (or an extension that knows what it's doing).

    The general idea is that where C code needs to execute Python
    code, it creates a frame for the Python code, setting its back
    pointer to the current frame. Then it swaps in the frame, signals
    the FrameDispatcher and gets out of the way. The C stack is now
    clean - the Python code can transfer control to any other frame
    (if an extension gives it the means to do so).

    In the vanilla case, this magic can be hidden from the programmer
    (even, in most cases, from the Python-internals programmer). Many
    situations present another level of difficulty, however.

    The map builtin function involves two obstacles to this
    approach. It cannot simply construct a frame and get out of the
    way, not just because there's a loop involved, but each pass
    through the loop requires some "post" processing. In order to play
    well with others, Stackless constructs a frame object for map
    itself.

    Most recursions of the interpreter are not this complex, but
    fairly frequently, some "post" operations are required. Stackless
    does not fix these situations because of amount of code changes
    required. Instead, Stackless prohibits transfers out of a nested
    interpreter. While not ideal (and sometimes puzzling), this
    limitation is hardly crippling.

    For normal Python, the advantage to this approach is that C stack
    usage becomes much smaller and more predictable. Unbounded
    recursion in Python code becomes a memory error, instead of a
    stack error (and thus, in non-Cupertino operating systems,
    something that can be recovered from).  The price, of course, is
    the added complexity that comes from transforming recursions of
    the byte-code interpreter loop into a higher order loop (and the
    attendant bookkeeping involved).

    The big advantage comes from realizing that the Python stack is
    really a tree, and the frame dispatcher can transfer control
    freely between leaf nodes of the tree, thus allowing things like
    microthreads and coroutines.

    [1] http://www.stackless.com
    [2] http://world.std.com/~wware/uthread.html
    [3] Demo/threads/Generator.py in the source distribution
    [4] http://www.stackless.com/coroutines.tim.peters.html

Python semi-coroutines

Since this has come up a couple of times, I finally dug thru
my files and found a few traces of this experiment and put
the files on <http://galen.med.virginia.edu/~sdm7g/Python/CO>

BTW: I considered the experiment a partial failure, because it
would not work to implement full-coroutines ( because ceval.c
is recursive and some of the state is implicit in the C calling
stack. ) but I believe it did successfully implement semi-coroutines 
( i.e. Icon-like generators ) -- which seems to be all that
some folks want. 

The mods were done to python-1.0.2 
I haven't looked at how hard this would be to update to 1.5.1 

--- README --- 

There were experimental mods to python-1.0.2 to make frameobjects
into resumable continuations. ( put here for the curious and just
in case anyone wants to try a similar experiment with a more
current release. )

the files modified were frameobject.h and ceval.c

A new opcode was added for SUSPEND, but no new suspend statement
was added to the parser. I used a python disassembler/assembler
to change specific RETURN opcodes into SUSPENDs.
( various .py files here were hacked in that manner and used for testing.
)

co.txt & cont.semantics were the beginning of some notes on
various thread control issues.

py-co is part of the mailing list correspondence between me, Tim and Guido
that started me on that experiment.

More notes on this later when I find time.

- Steve Majewski <sdm7g@Virginia.EDU>
   "name" || seq()
   c := create ("name" || seq())
   tempvar_name := @c
   c := ^c
dave &
bob
dave &
bob
wait
file=$1
shift
for dest in "$@"; do
    cp $file $dest
done
file=$1
shift
for dest in "$@"; do
    cp $file $dest &
done
wait
dc |&

while read line'?adc> '; do
    print -p "$(alg2rpn $line)"
    read -p answer
    print "    = $answer"
done
$ adc
adc> 2 + 3
    = 5
adc> (7 * 8) + 54
    = 110
adc> ^D
$
{ while read line'?adc> '; do
      print "$(alg2rpn $line)"
  done 
} | dc
PROCEDURE NEWPROCESS(proc: PROC;
                     addr: ADDRESS; size: CARDINAL;
                     VAR new: ADDRESS);
PROCEDURE TRANSFER(VAR source, destination: ADDRESS);

Softpanorama May the source be with you, but remember the KISS principle ;-)
	Contents	Bulletin	Scripting in shell and Perl	Network troubleshooting	History	Humor

News	See Also	Recommended Books	Recommended Links	Recommended Papers	Tutorials
Algorithms	Coroutines	*Braces Problem*	IDE environments	Jython
Debugging in Python	Tips	Quotes	Humor	Random Findings	Etc

Top updates <p>Your browser does not support iframes.</p>
Bulletin	Latest	Past week	Past month	Google Search

acknowledgments.txt	FOLKS WHO HAVE MADE THIS BOOK BETTER
intro.txt	INTRODUCTION
chap1.txt	PYTHON BASICS
chap2.txt	BASIC STRING OPERATIONS
chap3.txt	REGULAR EXPRESSIONS
chap4.txt	PARSERS AND STATE-MACHINES
chap5.txt	INTERNET TOOLS AND TECHNIQUES
appendix_a.txt	A SELECTIVE AND IMPRESSIONISTIC SHORT REVIEW OF PYTHON
appendix_b.txt	A DATA COMPRESSION PRIMER
appendix_c.txt	UNDERSTANDING UNICODE
appendix_d.txt	A STATE-MACHINE FOR ADDING MARKUP TO TEXT
glossary.txt	GLOSSARY TERMS

searchEnterpriseLinux:	Are there situations in which it's better to use Perl than Python, or vice versa?
van Rossum:	They're both competing for the same niche, in some cases. If all you need to do is simple text processing, you might use Perl. Python, much more than Perl, encourages clean coding habits to make it easy for other people to follow what you are doing. I've seen a lot of people who were developing in Perl moving to Python because it's easier to use. Where Python wins is when you have users who are not very sophisticated but have to write some code. An example would be in educational settings, where you have students who have no prior programming experience. Also, Python seems to work better when you have to write a large system, working together in a large group of developers, and when you want the lifetime of your program to be long.

searchEnterpriseLinux:	In what situations would people use Java instead of Python?
van Rossum:	Java and Python have quite different characteristics. Java is sophisticated language. It takes a longer time to learn Java than Python. With Java, you have to get used to the compile-edit-run cycle of software development. Python is a lighter-weight language. There is less to learn to get started. You can hit the ground running when you have only a few days of Python training under your belt. Python and Java can be used in the same project. Python would be used for higher level control of an application, and Java would be used for implementing the lower level that needs to run relatively efficiently. Another place where Python is used is an extension language. An example of that is Object Domain, a UML editing tool written in Java. It uses Python as an extension language.

searchEnterpriseLinux:	Will use of Python in developing applications increase?
van Rossum:	It's going to increase dramatically. There's a definite need for a language that's as easy to use as Python. Python is used to teach students how to program, so that is creating a base of people who know and like the language.

Top Visited <p>Your browser does not support iframes.</p>
Bulletin	Latest	Past week	Past month	Google Search

Python -- Scripting language which supports coroutines

Old News ;-)

[Oct 21, 2012] Last File Manager freshmeat.net

Written in Python. The last version is LFM 2.3 dated May 2011. Codebase is below 10K lines.

[Oct 06, 2011] Text Processing in Python (a book)

[Apr 04, 2011] Scripting the Linux desktop, Part 1 Basics by Paul Ferrill

Jan 18, 2011 | developerWorks

Python and Ubuntu

[Dec 25, 2010] Linux Developers choose Python as Best Programming Language and ...

Such polls mainly reflect what industry is using, no so much the quality of the language. In other poll the best Linux distribution is Ubuntu, which is probably the most primitive among the major distributions available.

[Dec 25, 2010] Russ's Notes on Python

[Oct 09, 2010] free e-book

[Aug 18, 2009] THE NEXT SNAKE BY RAINER GRIMM

[Jun 20, 2009] A Python Client/Server Tutorial by Phillip Watts

[Jul 8, 2008] Python Call Graph 0.5.1 by Gerald Kaszuba

[Jun 23, 2008] freshmeat.net Project details for cfv

[Jun 20, 2008] BitRock Download Web Stacks

[Mar 12, 2008] Terminator - Multiple GNOME terminals in one window

Rewrite of screen in Python?

[Apr 3, 2007] Charming Python Python elegance and warts, Part 1

[Oct 26, 2006] ONLamp.com -- What's New in Python 2.5

It's clear that Python is under pressure from Ruby :-)

[Sep 30, 2006] Python 2.5 Release We are pleased to announce the release of Python 2.5 (FINAL), the final, production release of Python 2.5, on September 19th, 2006.

[Sept 20, 2006] Python 101 cheat sheet

[Jul 27, 2006] eWeek/Microsoft Ships Python on .Net Darryl K. Taft

Dig Deep into Python Internals by Gigi Sayfan Part 1 of 2

[Feb 20, 2006] freshmeat.net Project details for Meld

Dive Into Python Python for experienced programmers

Wing IDE for Python Python IDS that includes source browser and editor. The editor supports folding

Reference Manual Wing IDE Version 1.1.4 Pythod IDS that includes source broser and editor.

What's New in Python 2.2 -- generators is a very interesting feature of Python 2.2 that is essentially a co-routine.

Dive Into Python Dive Into Python is a free Python book for experienced programmers.

[Dec 28, 2001] http://www.dotfunk.com/projects/pylint/pylint.0.1.tar.gz

[Dec 20, 2001] Semi-coroutines in Python 2.2

PyChecker 0.4 PyChecker is a python source code checking tool to help you find common bugs. [Vaults of Parnassus]

My conclusion

[May 6, 2001] Zope for the Perl-CGI programmer by Michael Roberts (michael@vivtek.com) Zope is written in Python

Dave Warner wrote an excellent article on XML-RPC and Python here, using Meerkat as an example of a real XML-RPC server

[Nov 22, 2000]LinuxPR: PythonWorks Pro v1.1 - Now Also for Linux and Solaris!

[Nov 14, 2000] developerWorks Linux Charming Python -- Inside Python's implementations See also Slashdot Interviews With The Creators of Vyper and Stackless

Fast, Free Prototyping in Python by Andrea Provaglio

[Apr 02, 2000] Linux Gazette; Exploring parsing and virtual machines with Python

See also

Background

Discussion

Problems

Advantages

References

Preserving State:

See Also:

Task 8.3

Task 8.4

coroutines and continuations ( in Python? - long discussion )

Continuations

Lisp

Emacs Lisp