Here is what I am doing now and am having the same problem:

public boolean handleMouseEvent(MouseEvent e) {
        if (e.getButton() == MouseEvent.BUTTON3 && e.getID() ==
MouseEvent.MOUSE_CLICKED) {
            Point2D.Double p = new Point2D.Double(e.getX(), e.getY());

            Bag[] hitObjects = objectsHitBy(p);

            boolean empty = true;
            for (int i = 0; i < hitObjects.length; i++)
                if (hitObjects[i].numObjs > 0) {
                    empty = false;
                    break;
                }

            if (empty) {
                addPopup.setEvent(e);
                addPopup.show(this, e.getX(), e.getY());
                return true;
            } else {
                removePopup.removeAll();
                for (int x = hitObjects.length - 1; x >= 0; x--)
                    for (int i = 0; i < hitObjects[x].numObjs; i++) {
                        LocationWrapper wrapper = (LocationWrapper)
(hitObjects[x].objs[i]);
                        final Object obj = wrapper.getObject();
                        removePopup.add(new AbstractAction("Delete " +
obj.toString()) {
                            @Override
                            public void actionPerformed(ActionEvent e) {
                                synchronized (state.schedule) {
                                    IAgent agent = (IAgent) obj;
                                    if (agent instanceof Manager &&amp;
state.managers().length == 1) {
                                       
JOptionPane.showMessageDialog(InteractiveDisplay2D.this, "Need at least one
manager",
                                                "Manager Needed",
JOptionPane.ERROR_MESSAGE, null);
                                        return;
                                    }

                                    removeAgent(agent);
                                    if (agent instanceof Manager) {
                                       
ui.dataset.removeSeries(agent.toString() + " Profit");
                                        for (Restocker restocker :
((Manager) agent).getRestockers())
                                            removeAgent(restocker);
                                        for (CokeMachine machine :
((Manager) agent).getMachines())
                                            removeAgent(machine);
                                    }
                                }
                            }
                        });
                    }
            }
            removePopup.show(this, e.getX(), e.getY());
            return true;
        } else
            return super.handleMouseEvent(e);
    }

    private void removeAgent(IAgent agent) {
        for (IBehavior behavior : agent.getBehaviors())
            if (behavior.getStopper() != null)
                behavior.getStopper().stop();
        Manager[] managers = state.managers();
        for (Manager manager : managers) {
            if (agent instanceof Restocker)
                manager.getRestockers().remove(agent);
            else if (agent instanceof CokeMachine)
                manager.getMachines().remove(agent);
        }
        state.getAgents().remove(agent);
        state.agentGrid.remove(agent);
        ui.controller.refresh();
    }

IAgent and IBehavior are wrappers over MASON.