if(ord.getState() == IOrder.State.OPENED) {
   if(ord.getId()==frame.getPnlOrdersOrdIDModify()){//Find the Order to edit
     if(ord.getOpenPrice()!=frame.getPnlOrdersPriceModify()){//Compares OpenPrice if different modify OpenPrice
        try {
              ord.setOpenPrice(frame.getPnlOrdersPriceModify()); //Set OpenPrice existing order
              //ord.waitForUpdate(5000);
              } catch (JFException e) {
              e.printStackTrace();
              }                 
     }
   }
}

1. What JForex-API version do you use?
2. The code snippet does not suffice to detect the reason of the validation failure. Please provide an example program which replicates the case.

...
    if(ord.getOpenPrice()!=frame.getPnlOrdersPriceModify()){//Compares OpenPrice if different modify OpenPrice
        try {
              console.getOut().println("order amount: " + ord.getAmount());
              console.getOut().println("order openprice before change: " + ord.getOpenPrice());
              console.getOut().println("order SL: " + ord.getStopLossPrice());
              console.getOut().println("order TP: " + ord.getTakeProfitPrice());
              console.getOut().println("order is : " + (ord.isLong() ? "LONG" : "SHORT"));
              console.getOut().println("order openprice new: " + frame.getPnlOrdersPriceModify());
              ord.setOpenPrice(frame.getPnlOrdersPriceModify()); //Set OpenPrice existing order
              //ord.waitForUpdate(5000);
              } catch (JFException e) {
              e.printStackTrace();
              }                 

private IOrder order;

@Override
public void onStart(IContext context) throws JFException {
    engine = context.getEngine();
    console = context.getConsole();
    ITick tick = context.getHistory().getLastTick(Instrument.EURUSD);
    order = engine.submitOrder("MarketOrder"+System.currentTimeMillis(), Instrument.EURUSD, OrderCommand.BUY, 0.01);
    order.waitForUpdate(State.OPENED);
    try{
        //this should raise the exception as one can't change an opened MARKET order
        order.setOpenPrice(tick.getBid() + Instrument.EURUSD.getPipValue() * 5);
    } catch (JFException e){
        console.getErr().println(e);
    }
    order = engine.submitOrder("ConditionalOrder"+System.currentTimeMillis(), Instrument.EURUSD, OrderCommand.BUYSTOP, 0.01, Instrument.EURUSD.getPipValue() * 5);
    order.waitForUpdate(State.OPENED);
    try{
        order.setOpenPrice(tick.getBid() + Instrument.EURUSD.getPipValue() * 10);
    } catch (JFException e){
        console.getErr().println(e);
    }        
}

OneOrderSetOpenPriceMarketVsCond.java [1.87 KiB]
Downloaded 101 times

package singlejartest;

import com.dukascopy.api.system.IClient;
import com.dukascopy.dds2.greed.GreedClient;

public class VersionCheck {

   public static void main(String[] args) {
      System.out.println(
            GreedClient.class.getProtectionDomain().getCodeSource().getLocation()+ "\n" +
            IClient.class.getProtectionDomain().getCodeSource().getLocation()
            );      
   }

}