Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 78 additions & 11 deletions Lib/multiprocessing/synchronize.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,22 +125,89 @@ def _make_name():
return '%s-%s' % (process.current_process()._config['semprefix'],
next(SemLock._rand))

if sys.platform == 'darwin':
#
# Specific MacOSX Semaphore
#

class _MacOSXSemaphore(SemLock):
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wouldn't it be better to handle the SemLock class differently at the C level?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wouldn't it be better to handle the SemLock class differently at the C level?

As I mentioned in the problem, I'm waiting for feedback (from the Mac OS team ?) before going any further.

A C implementation is an option, even if it seems more complicated than Python.
That said, I'm available to go deep into the SemLock C code

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, since the SemLock class is implemented in C, it's preferrable to first patch it there IMO. But let's CC some people I know they are on macOS: @ned-deily @ronaldoussoren @freakboy3742.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can't profess any particular expertise in this area. However, given the underlying implementation is written in C, I agree that it makes some sense for the implementation of this workaround to also be in C (rather than have an underlying implementation that returns NotImplemented, wrapped by a layer that overrides that.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

rather than have an underlying implementation that returns NotImplemented, wrapped by a layer that overrides that.

The main idea is to listen to the acquire and release methods to update a shared counter, and to return the counter value when the get_value method is invoked.

I am going to open a new PR with a workaround written in C.

"""Dedicated class used only to workaround the missing
function 'sem_getvalue', when interpreter runs on MacOSX.
Add two shared counters for each [Bounded]Semaphore in order
to calculate the internal value of the semaphore,
when acquire and release operations are called.
"""

def __init__(self, kind, value, maxvalue, *, ctx):
if not isinstance(self, Semaphore):
raise TypeError("_MacOSXSemaphore can only be used "
"as base class of Semaphore class")
self._rlock = ctx.RLock()
self._count = ctx.Value('h', value, lock=self._rlock)
self._pending_acquires = ctx.Value('h', 0, lock=self._rlock)
super().__init__(kind, value, maxvalue, ctx=ctx)

def acquire(self, blocking=True, timeout=None):
with self._rlock:
self._pending_acquires.value += 1
if self._semlock.acquire(blocking, timeout):
with self._rlock:
self._pending_acquires.value -= 1
self._count.value -= 1
return True
with self._rlock:
self._pending_acquires.value -= 1
return False

def release(self):
if isinstance(self, BoundedSemaphore):
with self._rlock:
if self.get_value() + 1 > self._semlock.maxvalue:
raise ValueError(f"semaphore released too many times")
with self._rlock:
self._count.value += 1
self._semlock.release()

def get_value(self):
with self._rlock:
val = self._count.value - self._pending_acquires.value
return val if val > 0 else 0

def _make_methods(self):
# Do not call the `Semlock._make_methods` method,
# because that breaks the reference to the local
# `acquire` and `release` methods.
pass

def __setstate__(self, state):
self._count, self._pending_acquires, self._rlock, state \
= state[-3], state[-2], state[-1], state[:-3]
super().__setstate__(state)

def __getstate__(self) -> tuple:
return super().__getstate__() \
+ (self._count, self._pending_acquires, self._rlock)


_SemClass = _MacOSXSemaphore
else:
class _NotMacOSXSemaphore(SemLock):
def __init__(self, kind, value, maxvalue, *, ctx):
super().__init__(kind, value, maxvalue, ctx=ctx)

def get_value(self) -> int:
return self._semlock._get_value()

_SemClass = _NotMacOSXSemaphore

#
# Semaphore
#

class Semaphore(SemLock):
class Semaphore(_SemClass):

def __init__(self, value=1, *, ctx):
SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx)

def get_value(self):
'''Returns current value of Semaphore.

Raises NotImplementedError on Mac OSX
because of broken sem_getvalue().
'''
return self._semlock._get_value()
_SemClass.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx)

def __repr__(self):
try:
Expand All @@ -156,7 +223,7 @@ def __repr__(self):
class BoundedSemaphore(Semaphore):

def __init__(self, value=1, *, ctx):
SemLock.__init__(self, SEMAPHORE, value, value, ctx=ctx)
_SemClass.__init__(self, SEMAPHORE, value, value, ctx=ctx)

def __repr__(self):
try:
Expand Down
24 changes: 20 additions & 4 deletions Lib/test/_test_multiprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -1750,10 +1750,26 @@ def test_semaphore(self):
def test_bounded_semaphore(self):
sem = self.BoundedSemaphore(2)
self._test_semaphore(sem)
# Currently fails on OS/X
#if HAVE_GETVALUE:
# self.assertRaises(ValueError, sem.release)
# self.assertReturnsIfImplemented(2, get_value, sem)
self.assertRaises(ValueError, sem.release)
self.assertReturnsIfImplemented(2, get_value, sem)

@unittest.skipIf(sys.platform != 'darwin', 'Darwin only')
def test_detect_macosx_semaphore(self):
if self.TYPE != 'processes':
self.skipTest('test not appropriate for {}'.format(self.TYPE))

sem = self.Semaphore(2)
mro = sem.__class__.mro()
self.assertTrue(any('_MacOSXSemaphore' in cls.__name__ for cls in mro))

@unittest.skipIf(sys.platform != 'darwin', 'Darwin only')
def test_detect_macosx_boundedsemaphore(self):
if self.TYPE != 'processes':
self.skipTest('test not appropriate for {}'.format(self.TYPE))

sem = self.BoundedSemaphore(2)
mro = sem.__class__.mro()
self.assertTrue(any('_MacOSXSemaphore' in cls.__name__ for cls in mro))

def test_timeout(self):
if self.TYPE != 'processes':
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix the not implemented ``get_value`` for :class:`multiprocessing.Semaphore` on MacOSX
by adding a dedicated class in the ``synchronize.py`` file.
Loading